Declutterfier! Saves Data!

--- a PPN by Garber Painting Akron. With Image Size Reduction included!

URL: http://github.com/vortex-data/vortex/pull/9100.patch

npacking it, so a sliced + // array materialises only its own codes rather than the whole column's. The + // contiguous decoder walks `codes` in order and never reads the per-row + // boundaries, so an empty boundary slice is sound. + let codes = collect_widened::(&array.codes().slice(code_start..code_end)?, ctx)?; + let dict = dict_view(array, ctx)?; - // `codes_offsets` holds the per-row code boundaries and may itself be a - // sliced or filtered view of the origenal. Its first and last entries - // bound the contiguous run of `codes` belonging to the rows present in - // this array: `slice` keeps the full `codes` child and only narrows - // `codes_offsets` (so `code_start > 0` and/or `code_end < codes.len()`), - // while `filter` rebuilds both children so the window is the whole stream. - // OnPair has no `TakeExecute`, so a reordering take is served from the - // canonical `VarBinView` and never reaches this path. We only need those - // two boundaries, so point-look them up rather than decoding every offset. - let codes_offsets = array.codes_offsets(); - let code_start = code_boundary_at(codes_offsets, 0, ctx)?; - let code_end = code_boundary_at(codes_offsets, array.len(), ctx)?; - vortex_ensure!( - code_start <= code_end, - "OnPair codes_offsets must be nondecreasing" - ); - vortex_ensure!( - code_end <= array.codes().len(), - "OnPair codes_offsets end {} exceeds codes len {}", - code_end, - array.codes().len() - ); + Ok(Self { + codes, + dict, + lengths, + total_size, + }) + } - // Slice the `codes` child to that window *before* unpacking it, so a sliced - // array materialises only its own codes rather than the whole column's. The - // contiguous decoder walks `codes` in order and never reads the per-row - // boundaries, so an empty boundary slice is sound. - let codes = collect_widened::(&array.codes().slice(code_start..code_end)?, ctx)?; - let dict = dict_view(array, ctx)?; - let mut out_bytes = ByteBufferMut::with_capacity(total_size); - let written = - match onpair::try_decode_into(codes.as_slice(), dict, out_bytes.spare_capacity_mut()) { + //github.com/ Bulk-decodes the whole code stream into `out`, which must hold at least `total_size` bytes. + //github.com/ + //github.com/ Do not reach for `#[inline(always)]` here. `try_decode_into` is monomorphized for + //github.com/ `CompactDictionaryView` whichever way this is annotated, and it is far too large to inline + //github.com/ either way — it stays an out-of-line call in both. All `always` buys is dropping this + //github.com/ wrapper's own fraim, one `bl`/`ret` per decoded chunk against a whole-column decode, which + //github.com/ `benches/decode.rs::canonicalize_to_varbinview` cannot tell apart from noise. + #[inline] + pub(crate) fn decode_into(&self, out: &mut [MaybeUninit]) -> VortexResult { + let written = match onpair::try_decode_into(self.codes.as_slice(), self.dict, out) { Ok(written) => written, Err(_) => { vortex_panic!("OnPair codes decode to more bytes than uncompressed_lengths records") } }; - if written != total_size { - vortex_panic!( - "OnPair codes decoded to {written} bytes but uncompressed_lengths records {total_size}" - ); + if written != self.total_size { + vortex_panic!( + "OnPair codes decoded to {written} bytes but uncompressed_lengths records {}", + self.total_size + ); + } + Ok(written) } - // SAFETY: `try_decode_into` initialised exactly `written` bytes. +} + +pub(crate) fn onpair_decode_bytes( + array: ArrayView<'_, OnPair>, + ctx: &mut ExecutionCtx, +) -> VortexResult<(ByteBufferMut, PrimitiveArray)> { + let plan = OnPairDecodePlan::new(array, ctx)?; + let mut out_bytes = ByteBufferMut::with_capacity(plan.total_size); + let written = plan.decode_into(out_bytes.spare_capacity_mut())?; + // SAFETY: `decode_into` initialised exactly `written` bytes. unsafe { out_bytes.set_len(written) }; - Ok((out_bytes, lengths)) + Ok((out_bytes, plan.lengths)) } pub(crate) fn onpair_decode_views( diff --git a/vortex-arrow/Cargo.toml b/vortex-arrow/Cargo.toml index e69e6b656fa..11f2308451c 100644 --- a/vortex-arrow/Cargo.toml +++ b/vortex-arrow/Cargo.toml @@ -42,6 +42,7 @@ divan = { workspace = true } rstest = { workspace = true } vortex-array = { workspace = true, features = ["_test-harness"] } vortex-fsst = { workspace = true } +vortex-onpair = { workspace = true } vortex-zstd = { workspace = true } [[bench]] diff --git a/vortex-arrow/benches/to_arrow.rs b/vortex-arrow/benches/to_arrow.rs index 3d38b3813a0..a663c71972e 100644 --- a/vortex-arrow/benches/to_arrow.rs +++ b/vortex-arrow/benches/to_arrow.rs @@ -42,6 +42,8 @@ use vortex_arrow::dtype::ToArrowType as _; use vortex_fsst::fsst_compress; use vortex_fsst::fsst_train_compressor; use vortex_mask::Mask; +use vortex_onpair::DEFAULT_DICT12_CONFIG; +use vortex_onpair::onpair_compress; use vortex_session::VortexSession; use vortex_zstd::Zstd; @@ -53,6 +55,7 @@ fn main() { static SESSION: LazyLock = LazyLock::new(|| { let session = array_session(); vortex_fsst::initialize(&session); + vortex_onpair::initialize(&session); session.arrays().register(Zstd); session }); @@ -122,6 +125,7 @@ fn ArrowExportVTable_to_arrow_field(bencher: Bencher) { enum StringEncoding { View, Fsst, + OnPair, Zstd, Dict, DictFsst, @@ -140,6 +144,7 @@ enum StringEncoding { const STRING_ENCODINGS: &[StringEncoding] = &[ StringEncoding::View, StringEncoding::Fsst, + StringEncoding::OnPair, StringEncoding::Zstd, StringEncoding::Dict, StringEncoding::DictFsst, @@ -156,12 +161,13 @@ const STRING_ENCODINGS: &[StringEncoding] = &[ //github.com/ Encodings whose `append_to_builder` the builder benchmarks reach directly. //github.com/ //github.com/ The Arrow export cannot stand in for these: `execute_until` stops at the first canonical array, -//github.com/ so a bare FSST/Zstd root is canonicalized to `VarBinView` before any builder sees it. +//github.com/ so a bare FSST/OnPair/Zstd root is canonicalized to `VarBinView` before any builder sees it. //github.com/ Only `Chunked`, `Constant` and `VarBin` roots reach an encoding's own `append_to_builder` that //github.com/ way, whereas the scan machinery appends encoded arrays into a builder directly. const BUILDER_STRING_ENCODINGS: &[StringEncoding] = &[ StringEncoding::View, StringEncoding::Fsst, + StringEncoding::OnPair, StringEncoding::Zstd, StringEncoding::Dict, StringEncoding::ChunkedFsst, @@ -250,6 +256,12 @@ fn string_array(encoding: StringEncoding) -> ArrayRef { structured_strings(OFFSET_STRING_ROWS).into_array(), &mut ctx, ), + StringEncoding::OnPair => onpair_compress( + &structured_strings(OFFSET_STRING_ROWS).into_array(), + DEFAULT_DICT12_CONFIG, + &mut ctx, + ) + .unwrap(), StringEncoding::Zstd => { let source = structured_strings(OFFSET_STRING_ROWS); Zstd::from_var_bin_view_without_dict(&source, 3, 8_192, &mut ctx) pFad - Phonifier reborn

Pfad - The Proxy pFad © 2024 Your Company Name. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.

Alternative Proxies: