Skip to content

[wip] PatchedArray wiring: Part 2 #7145

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
a10y wants to merge 1 commit into
base: develop
Choose a base branch
from
Draft

[wip] PatchedArray wiring: Part 2 #7145

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions encodings/alp/public-api.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ pub fn vortex_alp::ALP::buffer(_array: vortex_array::array::view::ArrayView<'_,

pub fn vortex_alp::ALP::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option<alloc::string::String>

pub fn vortex_alp::ALP::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult<vortex_alp::ALPData>
pub fn vortex_alp::ALP::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult<vortex_array::array::erased::ArrayRef>

pub fn vortex_alp::ALP::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>

Expand Down Expand Up @@ -212,7 +212,7 @@ pub fn vortex_alp::ALPRD::buffer(_array: vortex_array::array::view::ArrayView<'_

pub fn vortex_alp::ALPRD::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option<alloc::string::String>

pub fn vortex_alp::ALPRD::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult<vortex_alp::ALPRDData>
pub fn vortex_alp::ALPRD::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult<vortex_array::array::erased::ArrayRef>

pub fn vortex_alp::ALPRD::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>

Expand Down
7 changes: 4 additions & 3 deletions encodings/alp/src/alp/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ impl VTable for ALP {
metadata: &Self::Metadata,
_buffers: &[BufferHandle],
children: &dyn ArrayChildren,
) -> VortexResult<ALPData> {
) -> VortexResult<ArrayRef> {
let encoded_ptype = match &dtype {
DType::Primitive(PType::F32, n) => DType::Primitive(PType::I32, *n),
DType::Primitive(PType::F64, n) => DType::Primitive(PType::I64, *n),
Expand All @@ -153,14 +153,15 @@ impl VTable for ALP {
})
.transpose()?;

ALPData::try_new(
Ok(ALPData::try_new(
encoded,
Exponents {
e: u8::try_from(metadata.exp_e)?,
f: u8::try_from(metadata.exp_f)?,
},
patches,
)
)?
.into_array())
}

fn slots(array: ArrayView<'_, Self>) -> &[Option<ArrayRef>] {
Expand Down
7 changes: 4 additions & 3 deletions encodings/alp/src/alp_rd/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ impl VTable for ALPRD {
metadata: &Self::Metadata,
_buffers: &[BufferHandle],
children: &dyn ArrayChildren,
) -> VortexResult<ALPRDData> {
) -> VortexResult<ArrayRef> {
if children.len() < 2 {
vortex_bail!(
"Expected at least 2 children for ALPRD encoding, found {}",
Expand Down Expand Up @@ -212,7 +212,7 @@ impl VTable for ALPRD {
})
.transpose()?;

ALPRDData::try_new(
Ok(ALPRDData::try_new(
dtype.clone(),
left_parts,
left_parts_dictionary,
Expand All @@ -224,7 +224,8 @@ impl VTable for ALPRD {
)
})?,
left_parts_patches,
)
)?
.into_array())
}

fn slots(array: ArrayView<'_, Self>) -> &[Option<ArrayRef>] {
Expand Down
26 changes: 13 additions & 13 deletions encodings/alp/src/alp_rd/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ use vortex_array::ExecutionCtx;
use vortex_array::IntoArray;
use vortex_array::patches::Patches;
use vortex_array::validity::Validity;
use vortex_fastlanes::bitpack_compress::BitPackedEncoder;
use vortex_fastlanes::bitpack_compress::bitpack_encode_unchecked;

mod array;
Expand Down Expand Up @@ -228,20 +229,19 @@ impl RDEncoder {

// Bit-pack down the encoded left-parts array that have been dictionary encoded.
let primitive_left = PrimitiveArray::new(left_parts, array.validity());
// SAFETY: by construction, all values in left_parts can be packed to left_bit_width.
let packed_left = unsafe {
bitpack_encode_unchecked(primitive_left, left_bit_width as _)
.vortex_expect("bitpack_encode_unchecked should succeed for left parts")
.into_array()
};

let packed_left = BitPackedEncoder::new(&primitive_left)
.with_bit_width(left_bit_width as _)
.pack()
.vortex_expect("bitpack_encode_unchecked should succeed for left parts")
.into_array()
.vortex_expect("Packed::into_array");
let primitive_right = PrimitiveArray::new(right_parts, Validity::NonNullable);
// SAFETY: by construction, all values in right_parts are right_bit_width + leading zeros.
let packed_right = unsafe {
bitpack_encode_unchecked(primitive_right, self.right_bit_width as _)
.vortex_expect("bitpack_encode_unchecked should succeed for right parts")
.into_array()
};
let packed_right = BitPackedEncoder::new(&primitive_right)
.with_bit_width(self.right_bit_width as _)
.pack()
.vortex_expect("bitpack_encode_unchecked should succeed for right parts")
.into_array()
.vortex_expect("Packed::into_array");

// Bit-pack the dict-encoded left-parts
// Bit-pack the right-parts
Expand Down
2 changes: 1 addition & 1 deletion encodings/bytebool/public-api.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ pub fn vortex_bytebool::ByteBool::buffer(array: vortex_array::array::view::Array

pub fn vortex_bytebool::ByteBool::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option<alloc::string::String>

pub fn vortex_bytebool::ByteBool::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult<vortex_bytebool::ByteBoolData>
pub fn vortex_bytebool::ByteBool::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult<vortex_array::array::erased::ArrayRef>

pub fn vortex_bytebool::ByteBool::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>

Expand Down
4 changes: 2 additions & 2 deletions encodings/bytebool/src/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ impl VTable for ByteBool {
_metadata: &Self::Metadata,
buffers: &[BufferHandle],
children: &dyn ArrayChildren,
) -> VortexResult<ByteBoolData> {
) -> VortexResult<ArrayRef> {
let validity = if children.is_empty() {
Validity::from(dtype.nullability())
} else if children.len() == 1 {
Expand All @@ -135,7 +135,7 @@ impl VTable for ByteBool {
}
let buffer = buffers[0].clone();

Ok(ByteBoolData::new(buffer, validity))
Ok(ByteBoolData::new(buffer, validity).into_array())
}

fn slots(array: ArrayView<'_, Self>) -> &[Option<ArrayRef>] {
Expand Down
2 changes: 1 addition & 1 deletion encodings/datetime-parts/public-api.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ pub fn vortex_datetime_parts::DateTimeParts::buffer(_array: vortex_array::array:

pub fn vortex_datetime_parts::DateTimeParts::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option<alloc::string::String>

pub fn vortex_datetime_parts::DateTimeParts::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult<vortex_datetime_parts::DateTimePartsData>
pub fn vortex_datetime_parts::DateTimeParts::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult<vortex_array::array::erased::ArrayRef>

pub fn vortex_datetime_parts::DateTimeParts::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>

Expand Down
4 changes: 2 additions & 2 deletions encodings/datetime-parts/src/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ impl VTable for DateTimeParts {
metadata: &Self::Metadata,
_buffers: &[BufferHandle],
children: &dyn ArrayChildren,
) -> VortexResult<DateTimePartsData> {
) -> VortexResult<ArrayRef> {
if children.len() != 3 {
vortex_bail!(
"Expected 3 children for datetime-parts encoding, found {}",
Expand All @@ -184,7 +184,7 @@ impl VTable for DateTimeParts {
len,
)?;

DateTimePartsData::try_new(dtype.clone(), days, seconds, subseconds)
Ok(DateTimePartsData::try_new(dtype.clone(), days, seconds, subseconds)?.into_array())
}

fn slots(array: ArrayView<'_, Self>) -> &[Option<ArrayRef>] {
Expand Down
2 changes: 1 addition & 1 deletion encodings/decimal-byte-parts/public-api.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ pub fn vortex_decimal_byte_parts::DecimalByteParts::buffer(_array: vortex_array:

pub fn vortex_decimal_byte_parts::DecimalByteParts::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option<alloc::string::String>

pub fn vortex_decimal_byte_parts::DecimalByteParts::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult<vortex_decimal_byte_parts::DecimalBytePartsData>
pub fn vortex_decimal_byte_parts::DecimalByteParts::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult<vortex_array::array::erased::ArrayRef>

pub fn vortex_decimal_byte_parts::DecimalByteParts::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>

Expand Down
4 changes: 2 additions & 2 deletions encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ impl VTable for DecimalByteParts {
metadata: &Self::Metadata,
_buffers: &[BufferHandle],
children: &dyn ArrayChildren,
) -> VortexResult<DecimalBytePartsData> {
) -> VortexResult<ArrayRef> {
let Some(decimal_dtype) = dtype.as_decimal_opt() else {
vortex_bail!("decoding decimal but given non decimal dtype {}", dtype)
};
Expand All @@ -151,7 +151,7 @@ impl VTable for DecimalByteParts {
"lower_part_count > 0 not currently supported"
);

DecimalBytePartsData::try_new(msp, *decimal_dtype)
Ok(DecimalBytePartsData::try_new(msp, *decimal_dtype)?.into_array())
}

fn slots(array: ArrayView<'_, Self>) -> &[Option<ArrayRef>] {
Expand Down
18 changes: 0 additions & 18 deletions encodings/fastlanes/benches/bitpacking_take.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,12 +160,6 @@ fn patched_take_10_stratified(bencher: Bencher) {
let uncompressed = PrimitiveArray::new(values, Validity::NonNullable);
let packed = bitpack_to_best_bit_width(&uncompressed).unwrap();

assert!(packed.patches().is_some());
assert_eq!(
packed.patches().unwrap().num_patches(),
NUM_EXCEPTIONS as usize
);

let indices = PrimitiveArray::from_iter((0..10).map(|i| i * 6_653));

bencher
Expand All @@ -185,12 +179,6 @@ fn patched_take_10_contiguous(bencher: Bencher) {
let uncompressed = PrimitiveArray::new(values, Validity::NonNullable);
let packed = bitpack_to_best_bit_width(&uncompressed).unwrap();

assert!(packed.patches().is_some());
assert_eq!(
packed.patches().unwrap().num_patches(),
NUM_EXCEPTIONS as usize
);

let indices = buffer![0..10].into_array();

bencher
Expand Down Expand Up @@ -249,12 +237,6 @@ fn patched_take_10k_contiguous_patches(bencher: Bencher) {
let uncompressed = PrimitiveArray::new(values, Validity::NonNullable);
let packed = bitpack_to_best_bit_width(&uncompressed).unwrap();

assert!(packed.patches().is_some());
assert_eq!(
packed.patches().unwrap().num_patches(),
NUM_EXCEPTIONS as usize
);

let indices =
PrimitiveArray::from_iter((BIG_BASE2..BIG_BASE2 + NUM_EXCEPTIONS).cycle().take(10000));

Expand Down
Loading