From 5359525ec01836a17460ce6f61abd86c84380bcb Mon Sep 17 00:00:00 2001 From: Connor Tsui Date: Mon, 9 Feb 2026 16:21:31 -0500 Subject: [PATCH 1/9] refactor `vortex-scalar` Signed-off-by: Connor Tsui --- vortex-scalar/src/arbitrary.rs | 121 +- vortex-scalar/src/arrow.rs | 593 +++++++++ vortex-scalar/src/arrow/mod.rs | 203 ---- vortex-scalar/src/arrow/tests.rs | 406 ------- vortex-scalar/src/cast.rs | 70 ++ vortex-scalar/src/convert/decimal.rs | 136 +++ vortex-scalar/src/convert/from_scalar.rs | 283 +++++ vortex-scalar/src/convert/into_scalar.rs | 160 +++ vortex-scalar/src/convert/mod.rs | 16 + vortex-scalar/src/convert/primitive.rs | 219 ++++ vortex-scalar/src/decimal/macros.rs | 52 - vortex-scalar/src/decimal/mod.rs | 13 - vortex-scalar/src/decimal/value.rs | 610 ---------- vortex-scalar/src/display.rs | 50 +- vortex-scalar/src/lib.rs | 31 +- vortex-scalar/src/null.rs | 152 --- vortex-scalar/src/primitive.rs | 1070 ----------------- vortex-scalar/src/proto.rs | 613 +++++++--- vortex-scalar/src/scalar.rs | 697 ++++------- vortex-scalar/src/scalar_value.rs | 445 ++++--- vortex-scalar/src/tests/casting.rs | 152 +-- vortex-scalar/src/tests/consistency.rs | 30 +- vortex-scalar/src/tests/nested.rs | 35 +- vortex-scalar/src/tests/nullability.rs | 32 +- vortex-scalar/src/tests/primitives.rs | 37 +- vortex-scalar/src/tests/round_trip.rs | 66 +- vortex-scalar/src/{ => typed_view}/binary.rs | 321 ++--- vortex-scalar/src/{ => typed_view}/bool.rs | 156 +-- .../src/typed_view/decimal/dvalue.rs | 233 ++++ vortex-scalar/src/typed_view/decimal/mod.rs | 13 + .../src/{ => typed_view}/decimal/scalar.rs | 36 +- .../src/{ => typed_view}/decimal/tests.rs | 321 ++++- .../src/{ => typed_view}/extension.rs | 91 +- vortex-scalar/src/{ => typed_view}/list.rs | 146 ++- vortex-scalar/src/typed_view/mod.rs | 34 + vortex-scalar/src/typed_view/primitive/mod.rs | 15 + .../typed_view/primitive/numeric_operator.rs | 49 + .../src/{ => typed_view/primitive}/pvalue.rs | 423 +------ .../src/typed_view/primitive/scalar.rs | 431 +++++++ .../src/typed_view/primitive/tests.rs | 761 ++++++++++++ vortex-scalar/src/{ => typed_view}/struct_.rs | 148 +-- vortex-scalar/src/{ => typed_view}/utf8.rs | 385 ++---- 42 files changed, 4878 insertions(+), 4977 deletions(-) create mode 100644 vortex-scalar/src/arrow.rs delete mode 100644 vortex-scalar/src/arrow/mod.rs delete mode 100644 vortex-scalar/src/arrow/tests.rs create mode 100644 vortex-scalar/src/cast.rs create mode 100644 vortex-scalar/src/convert/decimal.rs create mode 100644 vortex-scalar/src/convert/from_scalar.rs create mode 100644 vortex-scalar/src/convert/into_scalar.rs create mode 100644 vortex-scalar/src/convert/mod.rs create mode 100644 vortex-scalar/src/convert/primitive.rs delete mode 100644 vortex-scalar/src/decimal/macros.rs delete mode 100644 vortex-scalar/src/decimal/mod.rs delete mode 100644 vortex-scalar/src/decimal/value.rs delete mode 100644 vortex-scalar/src/null.rs delete mode 100644 vortex-scalar/src/primitive.rs rename vortex-scalar/src/{ => typed_view}/binary.rs (55%) rename vortex-scalar/src/{ => typed_view}/bool.rs (69%) create mode 100644 vortex-scalar/src/typed_view/decimal/dvalue.rs create mode 100644 vortex-scalar/src/typed_view/decimal/mod.rs rename vortex-scalar/src/{ => typed_view}/decimal/scalar.rs (93%) rename vortex-scalar/src/{ => typed_view}/decimal/tests.rs (75%) rename vortex-scalar/src/{ => typed_view}/extension.rs (83%) rename vortex-scalar/src/{ => typed_view}/list.rs (84%) create mode 100644 vortex-scalar/src/typed_view/mod.rs create mode 100644 vortex-scalar/src/typed_view/primitive/mod.rs create mode 100644 vortex-scalar/src/typed_view/primitive/numeric_operator.rs rename vortex-scalar/src/{ => typed_view/primitive}/pvalue.rs (57%) create mode 100644 vortex-scalar/src/typed_view/primitive/scalar.rs create mode 100644 vortex-scalar/src/typed_view/primitive/tests.rs rename vortex-scalar/src/{ => typed_view}/struct_.rs (85%) rename vortex-scalar/src/{ => typed_view}/utf8.rs (58%) diff --git a/vortex-scalar/src/arbitrary.rs b/vortex-scalar/src/arbitrary.rs index 45e4d9e85ec..ae98bf8ff7f 100644 --- a/vortex-scalar/src/arbitrary.rs +++ b/vortex-scalar/src/arbitrary.rs @@ -7,7 +7,6 @@ //! It is used by the fuzzer to test the correctness of the scalar value implementation. use std::iter; -use std::sync::Arc; use arbitrary::Result; use arbitrary::Unstructured; @@ -19,60 +18,90 @@ use vortex_dtype::NativeDecimalType; use vortex_dtype::PType; use vortex_dtype::half::f16; use vortex_dtype::match_each_decimal_value_type; +use vortex_error::VortexExpect; use crate::DecimalValue; -use crate::InnerScalarValue; use crate::PValue; use crate::Scalar; use crate::ScalarValue; -/// Generate an arbitrary scalar value of the given data type. +/// Generates an arbitrary [`Scalar`] of the given [`DType`]. +/// +/// # Errors +/// +/// Returns an error if the underlying arbitrary generation fails. pub fn random_scalar(u: &mut Unstructured, dtype: &DType) -> Result { - Ok(Scalar::new(dtype.clone(), random_scalar_value(u, dtype)?)) -} + // For nullable types, return null ~25% of the time. This is just to make sure we don't generate + // too few nulls. + if dtype.is_nullable() && u.ratio(1, 4)? { + return Ok(Scalar::null(dtype.clone())); + } -fn random_scalar_value(u: &mut Unstructured, dtype: &DType) -> Result { - match dtype { - DType::Null => Ok(ScalarValue(InnerScalarValue::Null)), - DType::Bool(_) => Ok(ScalarValue(InnerScalarValue::Bool(u.arbitrary()?))), - DType::Primitive(p, _) => Ok(ScalarValue(InnerScalarValue::Primitive(random_pvalue( - u, p, - )?))), - DType::Decimal(decimal_type, _) => random_decimal(u, decimal_type), - DType::Utf8(_) => Ok(ScalarValue(InnerScalarValue::BufferString(Arc::new( - BufferString::from(u.arbitrary::()?), - )))), - DType::Binary(_) => Ok(ScalarValue(InnerScalarValue::Buffer(Arc::new( - ByteBuffer::from(u.arbitrary::>()?), - )))), - DType::Struct(sdt, _) => Ok(ScalarValue(InnerScalarValue::List( - sdt.fields() - .map(|d| random_scalar_value(u, &d)) - .collect::>>()? - .into(), - ))), - DType::List(edt, _) => Ok(ScalarValue(InnerScalarValue::List( - iter::from_fn(|| { - // Creates `Some(_)` with 1/4 probability. - u.arbitrary() - .unwrap_or(false) - .then(|| random_scalar_value(u, edt)) - }) - .collect::>>()? - .into(), - ))), - DType::FixedSizeList(edt, size, _) => Ok(ScalarValue(InnerScalarValue::List( - (0..*size) - .map(|_| random_scalar_value(u, edt)) - .collect::>>()? - .into(), - ))), + Ok(match dtype { + DType::Null => Scalar::null(dtype.clone()), + DType::Bool(_) => Scalar::try_new(dtype.clone(), Some(ScalarValue::Bool(u.arbitrary()?))) + .vortex_expect("unable to construct random `Scalar`_"), + DType::Primitive(p, _) => Scalar::try_new( + dtype.clone(), + Some(ScalarValue::Primitive(random_pvalue(u, p)?)), + ) + .vortex_expect("unable to construct random `Scalar`_"), + DType::Decimal(decimal_type, _) => { + Scalar::try_new(dtype.clone(), Some(random_decimal(u, decimal_type)?)) + .vortex_expect("unable to construct random `Scalar`_") + } + DType::Utf8(_) => Scalar::try_new( + dtype.clone(), + Some(ScalarValue::Utf8(BufferString::from( + u.arbitrary::()?, + ))), + ) + .vortex_expect("unable to construct random `Scalar`_"), + DType::Binary(_) => Scalar::try_new( + dtype.clone(), + Some(ScalarValue::Binary(ByteBuffer::from( + u.arbitrary::>()?, + ))), + ) + .vortex_expect("unable to construct random `Scalar`_"), + DType::Struct(sdt, _) => Scalar::try_new( + dtype.clone(), + Some(ScalarValue::List( + sdt.fields() + .map(|d| random_scalar(u, &d).map(|s| s.into_value())) + .collect::>>()?, + )), + ) + .vortex_expect("unable to construct random `Scalar`_"), + DType::List(edt, _) => Scalar::try_new( + dtype.clone(), + Some(ScalarValue::List( + iter::from_fn(|| { + // Generate elements with 1/4 probability. + u.arbitrary() + .unwrap_or(false) + .then(|| random_scalar(u, edt).map(|s| s.into_value())) + }) + .collect::>>()?, + )), + ) + .vortex_expect("unable to construct random `Scalar`_"), + DType::FixedSizeList(edt, size, _) => Scalar::try_new( + dtype.clone(), + Some(ScalarValue::List( + (0..*size) + .map(|_| random_scalar(u, edt).map(|s| s.into_value())) + .collect::>>()?, + )), + ) + .vortex_expect("unable to construct random `Scalar`_"), DType::Extension(..) => { unreachable!("Can't yet generate arbitrary scalars for ext dtype") } - } + }) } +/// Generates an arbitrary [`PValue`] for the given [`PType`]. fn random_pvalue(u: &mut Unstructured, ptype: &PType) -> Result { Ok(match ptype { PType::U8 => PValue::U8(u.arbitrary()?), @@ -89,7 +118,11 @@ fn random_pvalue(u: &mut Unstructured, ptype: &PType) -> Result { }) } -/// Generate an arbitrary decimal scalar confined to the given bounds of precision and scale. +/// Generates an arbitrary decimal scalar confined to the given bounds of precision and scale. +/// +/// # Errors +/// +/// Returns an error if the underlying arbitrary generation fails. pub fn random_decimal(u: &mut Unstructured, decimal_type: &DecimalDType) -> Result { let precision = decimal_type.precision(); let value = match_each_decimal_value_type!( @@ -101,5 +134,5 @@ pub fn random_decimal(u: &mut Unstructured, decimal_type: &DecimalDType) -> Resu } ); - Ok(ScalarValue(InnerScalarValue::Decimal(value))) + Ok(ScalarValue::Decimal(value)) } diff --git a/vortex-scalar/src/arrow.rs b/vortex-scalar/src/arrow.rs new file mode 100644 index 00000000000..46989b71f08 --- /dev/null +++ b/vortex-scalar/src/arrow.rs @@ -0,0 +1,593 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Conversions between [`Scalar`] and Arrow scalar types. + +use std::sync::Arc; + +use arrow_array::Scalar as ArrowScalar; +use arrow_array::*; +use vortex_dtype::DType; +use vortex_dtype::PType; +use vortex_dtype::datetime::AnyTemporal; +use vortex_dtype::datetime::TemporalMetadata; +use vortex_dtype::datetime::TimeUnit; +use vortex_error::VortexError; +use vortex_error::vortex_bail; +use vortex_error::vortex_err; + +use crate::BinaryScalar; +use crate::BoolScalar; +use crate::DecimalScalar; +use crate::DecimalValue; +use crate::ExtScalar; +use crate::PrimitiveScalar; +use crate::Scalar; +use crate::Utf8Scalar; + +/// Arrow represents scalars as single-element arrays. This constant is the length of those arrays. +const SCALAR_ARRAY_LEN: usize = 1; + +/// Converts an optional value to an Arrow scalar array. +macro_rules! value_to_arrow_scalar { + ($V:expr, $AR:ty) => { + Ok(std::sync::Arc::new( + $V.map(<$AR>::new_scalar) + .unwrap_or_else(|| arrow_array::Scalar::new(<$AR>::new_null(SCALAR_ARRAY_LEN))), + )) + }; +} + +/// Converts an optional timestamp value to an Arrow scalar array. +macro_rules! timestamp_to_arrow_scalar { + ($V:expr, $TZ:expr, $AR:ty) => {{ + let array = match $V { + Some(v) => <$AR>::new_scalar(v).into_inner(), + None => <$AR>::new_null(SCALAR_ARRAY_LEN), + } + .with_timezone_opt($TZ); + Ok(Arc::new(ArrowScalar::new(array))) + }}; +} + +impl TryFrom<&Scalar> for Arc { + type Error = VortexError; + + fn try_from(value: &Scalar) -> Result, Self::Error> { + match value.dtype() { + DType::Null => Ok(Arc::new(NullArray::new(SCALAR_ARRAY_LEN))), + DType::Bool(_) => bool_to_arrow(value.as_bool()), + DType::Primitive(..) => primitive_to_arrow(value.as_primitive()), + DType::Decimal(..) => decimal_to_arrow(value.as_decimal()), + DType::Utf8(_) => utf8_to_arrow(value.as_utf8()), + DType::Binary(_) => binary_to_arrow(value.as_binary()), + DType::Struct(..) => unimplemented!("struct scalar conversion"), + DType::List(..) => unimplemented!("list scalar conversion"), + DType::FixedSizeList(..) => unimplemented!("fixed-size list scalar conversion"), + DType::Extension(..) => extension_to_arrow(value.as_extension()), + } + } +} + +/// Convert a [`BoolScalar`] to an Arrow [`Datum`]. +fn bool_to_arrow(scalar: BoolScalar<'_>) -> Result, VortexError> { + value_to_arrow_scalar!(scalar.value(), BooleanArray) +} + +/// Convert a [`PrimitiveScalar`] to an Arrow [`Datum`]. +fn primitive_to_arrow(scalar: PrimitiveScalar<'_>) -> Result, VortexError> { + match scalar.ptype() { + PType::U8 => value_to_arrow_scalar!(scalar.typed_value(), UInt8Array), + PType::U16 => value_to_arrow_scalar!(scalar.typed_value(), UInt16Array), + PType::U32 => value_to_arrow_scalar!(scalar.typed_value(), UInt32Array), + PType::U64 => value_to_arrow_scalar!(scalar.typed_value(), UInt64Array), + PType::I8 => value_to_arrow_scalar!(scalar.typed_value(), Int8Array), + PType::I16 => value_to_arrow_scalar!(scalar.typed_value(), Int16Array), + PType::I32 => value_to_arrow_scalar!(scalar.typed_value(), Int32Array), + PType::I64 => value_to_arrow_scalar!(scalar.typed_value(), Int64Array), + PType::F16 => value_to_arrow_scalar!(scalar.typed_value(), Float16Array), + PType::F32 => value_to_arrow_scalar!(scalar.typed_value(), Float32Array), + PType::F64 => value_to_arrow_scalar!(scalar.typed_value(), Float64Array), + } +} + +/// Convert a [`DecimalScalar`] to an Arrow [`Datum`]. +fn decimal_to_arrow(scalar: DecimalScalar<'_>) -> Result, VortexError> { + // TODO(joe): Replace with decimal32, etc. once Arrow supports them. + match scalar.decimal_value() { + Some(DecimalValue::I8(v)) => Ok(Arc::new(Decimal128Array::new_scalar(v as i128))), + Some(DecimalValue::I16(v)) => Ok(Arc::new(Decimal128Array::new_scalar(v as i128))), + Some(DecimalValue::I32(v)) => Ok(Arc::new(Decimal128Array::new_scalar(v as i128))), + Some(DecimalValue::I64(v)) => Ok(Arc::new(Decimal128Array::new_scalar(v as i128))), + Some(DecimalValue::I128(v128)) => Ok(Arc::new(Decimal128Array::new_scalar(v128))), + Some(DecimalValue::I256(v256)) => Ok(Arc::new(Decimal256Array::new_scalar(v256.into()))), + None => Ok(Arc::new(arrow_array::Scalar::new( + Decimal128Array::new_null(SCALAR_ARRAY_LEN), + ))), + } +} + +/// Convert a [`Utf8Scalar`] to an Arrow [`Datum`]. +fn utf8_to_arrow(scalar: Utf8Scalar<'_>) -> Result, VortexError> { + value_to_arrow_scalar!(scalar.value(), StringViewArray) +} + +/// Convert a [`BinaryScalar`] to an Arrow [`Datum`]. +fn binary_to_arrow(scalar: BinaryScalar<'_>) -> Result, VortexError> { + value_to_arrow_scalar!(scalar.value(), BinaryViewArray) +} + +/// Convert an [`ExtScalar`] to an Arrow [`Datum`]. +/// +/// Currently only temporal extension types (timestamps, dates, and times) are supported. +fn extension_to_arrow(scalar: ExtScalar<'_>) -> Result, VortexError> { + let ext_dtype = scalar.ext_dtype(); + let Some(temporal) = ext_dtype.metadata_opt::() else { + vortex_bail!( + "Cannot convert extension scalar {} to Arrow", + ext_dtype.id() + ) + }; + + let storage_scalar = scalar.storage(); + let primitive = storage_scalar + .as_primitive_opt() + .ok_or_else(|| vortex_err!("Expected primitive scalar"))?; + + match temporal { + TemporalMetadata::Timestamp(unit, tz) => { + let value = primitive.as_::(); + match unit { + TimeUnit::Nanoseconds => { + timestamp_to_arrow_scalar!(value, tz.clone(), TimestampNanosecondArray) + } + TimeUnit::Microseconds => { + timestamp_to_arrow_scalar!(value, tz.clone(), TimestampMicrosecondArray) + } + TimeUnit::Milliseconds => { + timestamp_to_arrow_scalar!(value, tz.clone(), TimestampMillisecondArray) + } + TimeUnit::Seconds => { + timestamp_to_arrow_scalar!(value, tz.clone(), TimestampSecondArray) + } + TimeUnit::Days => { + vortex_bail!("Unsupported TimeUnit {unit} for {}", ext_dtype.id()) + } + } + } + TemporalMetadata::Date(unit) => match unit { + TimeUnit::Milliseconds => { + value_to_arrow_scalar!(primitive.as_::(), Date64Array) + } + TimeUnit::Days => { + value_to_arrow_scalar!(primitive.as_::(), Date32Array) + } + TimeUnit::Nanoseconds | TimeUnit::Microseconds | TimeUnit::Seconds => { + vortex_bail!("Unsupported TimeUnit {unit} for {}", ext_dtype.id()) + } + }, + TemporalMetadata::Time(unit) => match unit { + TimeUnit::Nanoseconds => { + value_to_arrow_scalar!(primitive.as_::(), Time64NanosecondArray) + } + TimeUnit::Microseconds => { + value_to_arrow_scalar!(primitive.as_::(), Time64MicrosecondArray) + } + TimeUnit::Milliseconds => { + value_to_arrow_scalar!(primitive.as_::(), Time32MillisecondArray) + } + TimeUnit::Seconds => { + value_to_arrow_scalar!(primitive.as_::(), Time32SecondArray) + } + TimeUnit::Days => { + vortex_bail!("Unsupported TimeUnit {unit} for {}", ext_dtype.id()) + } + }, + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow_array::Datum; + use rstest::rstest; + use vortex_dtype::DType; + use vortex_dtype::DecimalDType; + use vortex_dtype::NativeDType; + use vortex_dtype::Nullability; + use vortex_dtype::PType; + use vortex_dtype::datetime::Date; + use vortex_dtype::datetime::Time; + use vortex_dtype::datetime::TimeUnit; + use vortex_dtype::datetime::Timestamp; + use vortex_dtype::datetime::TimestampOptions; + use vortex_dtype::extension::ExtDTypeVTable; + use vortex_error::VortexResult; + use vortex_error::vortex_bail; + + use crate::DecimalValue; + use crate::Scalar; + + #[test] + fn test_null_scalar_to_arrow() { + let scalar = Scalar::null(DType::Null); + let result = Arc::::try_from(&scalar); + assert!(result.is_ok()); + } + + #[test] + fn test_bool_scalar_to_arrow() { + let scalar = Scalar::bool(true, Nullability::NonNullable); + let result = Arc::::try_from(&scalar); + assert!(result.is_ok()); + } + + #[test] + fn test_null_bool_scalar_to_arrow() { + let scalar = Scalar::null(bool::dtype().as_nullable()); + let result = Arc::::try_from(&scalar); + assert!(result.is_ok()); + } + + #[test] + fn test_primitive_u8_to_arrow() { + let scalar = Scalar::primitive(42u8, Nullability::NonNullable); + let result = Arc::::try_from(&scalar); + assert!(result.is_ok()); + } + + #[test] + fn test_primitive_u16_to_arrow() { + let scalar = Scalar::primitive(1000u16, Nullability::NonNullable); + let result = Arc::::try_from(&scalar); + assert!(result.is_ok()); + } + + #[test] + fn test_primitive_u32_to_arrow() { + let scalar = Scalar::primitive(100000u32, Nullability::NonNullable); + let result = Arc::::try_from(&scalar); + assert!(result.is_ok()); + } + + #[test] + fn test_primitive_u64_to_arrow() { + let scalar = Scalar::primitive(10000000000u64, Nullability::NonNullable); + let result = Arc::::try_from(&scalar); + assert!(result.is_ok()); + } + + #[test] + fn test_primitive_i8_to_arrow() { + let scalar = Scalar::primitive(-42i8, Nullability::NonNullable); + let result = Arc::::try_from(&scalar); + assert!(result.is_ok()); + } + + #[test] + fn test_primitive_i16_to_arrow() { + let scalar = Scalar::primitive(-1000i16, Nullability::NonNullable); + let result = Arc::::try_from(&scalar); + assert!(result.is_ok()); + } + + #[test] + fn test_primitive_i32_to_arrow() { + let scalar = Scalar::primitive(-100000i32, Nullability::NonNullable); + let result = Arc::::try_from(&scalar); + assert!(result.is_ok()); + } + + #[test] + fn test_primitive_i64_to_arrow() { + let scalar = Scalar::primitive(-10000000000i64, Nullability::NonNullable); + let result = Arc::::try_from(&scalar); + assert!(result.is_ok()); + } + + #[test] + fn test_primitive_f16_to_arrow() { + use vortex_dtype::half::f16; + + let scalar = Scalar::primitive(f16::from_f32(1.234), Nullability::NonNullable); + let result = Arc::::try_from(&scalar); + assert!(result.is_ok()); + } + + #[test] + fn test_primitive_f32_to_arrow() { + let scalar = Scalar::primitive(1.234f32, Nullability::NonNullable); + let result = Arc::::try_from(&scalar); + assert!(result.is_ok()); + } + + #[test] + fn test_primitive_f64_to_arrow() { + let scalar = Scalar::primitive(1.234567890123f64, Nullability::NonNullable); + let result = Arc::::try_from(&scalar); + assert!(result.is_ok()); + } + + #[test] + fn test_null_primitive_to_arrow() { + let scalar = Scalar::null(i32::dtype().as_nullable()); + let result = Arc::::try_from(&scalar); + assert!(result.is_ok()); + } + + #[test] + fn test_utf8_scalar_to_arrow() { + let scalar = Scalar::utf8("hello world".to_string(), Nullability::NonNullable); + let result = Arc::::try_from(&scalar); + assert!(result.is_ok()); + } + + #[test] + fn test_null_utf8_scalar_to_arrow() { + let scalar = Scalar::null(String::dtype().as_nullable()); + let result = Arc::::try_from(&scalar); + assert!(result.is_ok()); + } + + #[test] + fn test_binary_scalar_to_arrow() { + let data = vec![1u8, 2, 3, 4, 5]; + let scalar = Scalar::binary(data, Nullability::NonNullable); + let result = Arc::::try_from(&scalar); + assert!(result.is_ok()); + } + + #[test] + fn test_null_binary_scalar_to_arrow() { + let scalar = Scalar::null(DType::Binary(Nullability::Nullable)); + let result = Arc::::try_from(&scalar); + assert!(result.is_ok()); + } + + #[test] + fn test_decimal_scalars_to_arrow() { + // Test various decimal value types + let decimal_dtype = DecimalDType::new(5, 2); + + let scalar_i8 = Scalar::decimal( + DecimalValue::I8(100), + decimal_dtype, + Nullability::NonNullable, + ); + assert!(Arc::::try_from(&scalar_i8).is_ok()); + + let scalar_i16 = Scalar::decimal( + DecimalValue::I16(10000), + decimal_dtype, + Nullability::NonNullable, + ); + assert!(Arc::::try_from(&scalar_i16).is_ok()); + + let scalar_i32 = Scalar::decimal( + DecimalValue::I32(99999), + decimal_dtype, + Nullability::NonNullable, + ); + assert!(Arc::::try_from(&scalar_i32).is_ok()); + + let scalar_i64 = Scalar::decimal( + DecimalValue::I64(99999), + decimal_dtype, + Nullability::NonNullable, + ); + assert!(Arc::::try_from(&scalar_i64).is_ok()); + + let scalar_i128 = Scalar::decimal( + DecimalValue::I128(99999), + decimal_dtype, + Nullability::NonNullable, + ); + assert!(Arc::::try_from(&scalar_i128).is_ok()); + + // Test i256 + use vortex_dtype::i256; + let value_i256 = i256::from_i128(99999); + let scalar_i256 = Scalar::decimal( + DecimalValue::I256(value_i256), + decimal_dtype, + Nullability::NonNullable, + ); + assert!(Arc::::try_from(&scalar_i256).is_ok()); + } + + #[test] + fn test_null_decimal_to_arrow() { + use vortex_dtype::DecimalDType; + + let decimal_dtype = DecimalDType::new(10, 2); + let scalar = Scalar::null(DType::Decimal(decimal_dtype, Nullability::Nullable)); + let result = Arc::::try_from(&scalar); + assert!(result.is_ok()); + } + + #[test] + #[should_panic(expected = "struct scalar conversion")] + fn test_struct_scalar_to_arrow_todo() { + use vortex_dtype::FieldDType; + use vortex_dtype::StructFields; + + let struct_dtype = DType::Struct( + StructFields::from_iter([( + "field1", + FieldDType::from(DType::Primitive(PType::I32, Nullability::NonNullable)), + )]), + Nullability::NonNullable, + ); + + let struct_scalar = Scalar::struct_( + struct_dtype, + vec![Scalar::primitive(42i32, Nullability::NonNullable)], + ); + Arc::::try_from(&struct_scalar).unwrap(); + } + + #[test] + #[should_panic(expected = "list scalar conversion")] + fn test_list_scalar_to_arrow_todo() { + let element_dtype = Arc::new(DType::Primitive(PType::I32, Nullability::NonNullable)); + let list_scalar = Scalar::list( + element_dtype, + vec![ + Scalar::primitive(1i32, Nullability::NonNullable), + Scalar::primitive(2i32, Nullability::NonNullable), + ], + Nullability::NonNullable, + ); + + Arc::::try_from(&list_scalar).unwrap(); + } + + #[test] + #[should_panic(expected = "Cannot convert extension scalar")] + fn test_non_temporal_extension_to_arrow_todo() { + use vortex_dtype::ExtID; + + #[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] + struct SomeExt; + impl ExtDTypeVTable for SomeExt { + type Metadata = String; + + fn id(&self) -> ExtID { + ExtID::new_ref("some_ext") + } + + fn serialize(&self, _options: &Self::Metadata) -> VortexResult> { + vortex_bail!("not implemented") + } + + fn deserialize(&self, _data: &[u8]) -> VortexResult { + vortex_bail!("not implemented") + } + + fn validate_dtype( + &self, + _options: &Self::Metadata, + _storage_dtype: &DType, + ) -> VortexResult<()> { + Ok(()) + } + } + + let scalar = Scalar::extension::( + "".into(), + Scalar::primitive(42i32, Nullability::NonNullable), + ); + + Arc::::try_from(&scalar).unwrap(); + } + + #[rstest] + #[case(TimeUnit::Nanoseconds, PType::I64, 123456789i64)] + #[case(TimeUnit::Microseconds, PType::I64, 123456789i64)] + #[case(TimeUnit::Milliseconds, PType::I32, 123456i64)] + #[case(TimeUnit::Seconds, PType::I32, 1234i64)] + fn test_temporal_time_to_arrow( + #[case] time_unit: TimeUnit, + #[case] ptype: PType, + #[case] value: i64, + ) { + let scalar = Scalar::extension::