Skip to content

Scalar truncation is implemented over scalar values instead of scalars and can operate in place #6414

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
robert3005 wants to merge 3 commits into
base: develop
Choose a base branch
from
+382 −322
Open

Scalar truncation is implemented over scalar values instead of scalars and can operate in place #6414

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f35cef5
Scalar truncation is implemented over scalar values instead of scalar…
robert3005 Feb 10, 2026
991e394
comments
robert3005 Feb 11, 2026
17406b8
api
robert3005 Feb 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions vortex-layout/public-api.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -960,9 +960,9 @@ pub const vortex_layout::layouts::zoned::MAX_IS_TRUNCATED: &str

pub const vortex_layout::layouts::zoned::MIN_IS_TRUNCATED: &str

pub fn vortex_layout::layouts::zoned::lower_bound(value: impl vortex_layout::layouts::zoned::builder::ScalarTruncation, max_length: usize) -> (vortex_scalar::scalar::Scalar, bool)
pub fn vortex_layout::layouts::zoned::lower_bound(value: core::option::Option<impl vortex_layout::layouts::zoned::builder::StatTruncation>, max_length: usize, nullability: vortex_dtype::nullability::Nullability) -> core::option::Option<(vortex_scalar::scalar::Scalar, bool)>

pub fn vortex_layout::layouts::zoned::upper_bound(value: impl vortex_layout::layouts::zoned::builder::ScalarTruncation, max_length: usize) -> (core::option::Option<vortex_scalar::scalar::Scalar>, bool)
pub fn vortex_layout::layouts::zoned::upper_bound(value: core::option::Option<impl vortex_layout::layouts::zoned::builder::StatTruncation>, max_length: usize, nullability: vortex_dtype::nullability::Nullability) -> core::option::Option<(vortex_scalar::scalar::Scalar, bool)>

pub type vortex_layout::layouts::SharedArrayFuture = futures_util::future::future::shared::Shared<futures_core::future::BoxFuture<'static, vortex_error::SharedVortexResult<vortex_array::array::ArrayRef>>>

Expand Down
101 changes: 50 additions & 51 deletions vortex-layout/src/layouts/flat/writer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,12 @@ use vortex_array::normalize::NormalizeOptions;
use vortex_array::normalize::Operation;
use vortex_array::serde::SerializeOptions;
use vortex_array::session::ArrayRegistry;
use vortex_array::stats::StatsSetRef;
use vortex_dtype::DType;
use vortex_error::VortexResult;
use vortex_error::vortex_bail;
use vortex_io::runtime::Handle;
use vortex_scalar::Scalar;

use crate::IntoLayout;
use crate::LayoutRef;
Expand Down Expand Up @@ -69,6 +71,22 @@ impl FlatLayoutStrategy {
}
}

fn truncate_scalar_stat<F: Fn(Scalar) -> Option<(Scalar, bool)>>(
statistics: StatsSetRef<'_>,
stat: Stat,
truncation: F,
) {
if let Some(sv) = statistics.get(stat) {
if let Some((truncated_value, truncated)) = truncation(sv.into_inner()) {
if truncated && let Some(v) = truncated_value.into_value() {
statistics.set(stat, Precision::Inexact(v));
}
} else {
statistics.clear(stat)
}
}
}

#[async_trait]
impl LayoutStrategy for FlatLayoutStrategy {
async fn write_stream(
Expand All @@ -80,7 +98,6 @@ impl LayoutStrategy for FlatLayoutStrategy {
_handle: Handle,
) -> VortexResult<LayoutRef> {
let ctx = ctx.clone();
let options = self.clone();
let Some(chunk) = stream.next().await else {
vortex_bail!("flat layout needs a single chunk");
};
Expand All @@ -89,60 +106,42 @@ impl LayoutStrategy for FlatLayoutStrategy {
let row_count = chunk.len() as u64;

match chunk.dtype() {
DType::Utf8(_) => {
if let Some(sv) = chunk.statistics().get(Stat::Min) {
let (value, truncated) = lower_bound(
sv.into_inner().as_utf8(),
options.max_variable_length_statistics_size,
);
if truncated && let Some(v) = value.into_value() {
chunk.statistics().set(Stat::Min, Precision::Inexact(v));
}
}

if let Some(sv) = chunk.statistics().get(Stat::Max) {
let (value, truncated) = upper_bound(
sv.into_inner().as_utf8(),
options.max_variable_length_statistics_size,
);
if let Some(upper_bound) = value {
if truncated && let Some(v) = upper_bound.into_value() {
chunk.statistics().set(Stat::Max, Precision::Inexact(v));
}
} else {
chunk.statistics().clear(Stat::Max)
}
}
DType::Utf8(n) => {
truncate_scalar_stat(chunk.statistics(), Stat::Min, |v| {
lower_bound(
v.into_value().map(|v| v.into_utf8()),
self.max_variable_length_statistics_size,
*n,
)
});
truncate_scalar_stat(chunk.statistics(), Stat::Max, |v| {
upper_bound(
v.into_value().map(|v| v.into_utf8()),
self.max_variable_length_statistics_size,
*n,
)
});
}
DType::Binary(_) => {
if let Some(sv) = chunk.statistics().get(Stat::Min) {
let (value, truncated) = lower_bound(
sv.into_inner().as_binary(),
options.max_variable_length_statistics_size,
);
if truncated && let Some(v) = value.into_value() {
chunk.statistics().set(Stat::Min, Precision::Inexact(v));
}
}

if let Some(sv) = chunk.statistics().get(Stat::Max) {
let (value, truncated) = upper_bound(
sv.into_inner().as_binary(),
options.max_variable_length_statistics_size,
);
if let Some(upper_bound) = value {
if truncated && let Some(v) = upper_bound.into_value() {
chunk.statistics().set(Stat::Max, Precision::Inexact(v));
}
} else {
chunk.statistics().clear(Stat::Max)
}
}
DType::Binary(n) => {
truncate_scalar_stat(chunk.statistics(), Stat::Min, |v| {
lower_bound(
v.into_value().map(|v| v.into_binary()),
self.max_variable_length_statistics_size,
*n,
)
});
truncate_scalar_stat(chunk.statistics(), Stat::Max, |v| {
upper_bound(
v.into_value().map(|v| v.into_binary()),
self.max_variable_length_statistics_size,
*n,
)
});
}
_ => {}
}

let chunk = if let Some(allowed) = &options.allowed_encodings {
let chunk = if let Some(allowed) = &self.allowed_encodings {
chunk.normalize(&mut NormalizeOptions {
allowed,
operation: Operation::Error,
Expand All @@ -155,7 +154,7 @@ impl LayoutStrategy for FlatLayoutStrategy {
&ctx,
&SerializeOptions {
offset: 0,
include_padding: options.include_padding,
include_padding: self.include_padding,
},
)?;
// there is at least the flatbuffer and the length
Expand Down
Loading
Loading