Route Arrow - DType extension conversion through the registry

Cargo.toml

@@ -106,6 +106,7 @@ async-lock = "3.4"
 async-stream = "0.3.6"
 async-trait = "0.1.89"
 base16ct = "1.0.0"
+base64 = "0.22"
 bigdecimal = "0.4.8"
 bindgen = "0.72.0"
 bit-vec = "0.9.0"

vortex-array/Cargo.toml

@@ -33,6 +33,7 @@ arrow-schema = { workspace = true }
 arrow-select = { workspace = true }
 arrow-string = { workspace = true }
 async-lock = { workspace = true }
+base64 = { workspace = true }
 bytes = { workspace = true }
 cfg-if = { workspace = true }
 cudarc = { workspace = true, optional = true }

vortex-array/public-api.lock

@@ -8472,26 +8472,38 @@ pub trait vortex_array::dtype::arrow::FromArrowType<T>: core::marker::Sized
 
 pub fn vortex_array::dtype::arrow::FromArrowType::from_arrow(value: T) -> Self
 
+pub fn vortex_array::dtype::arrow::FromArrowType::from_arrow_with_session(value: T, _session: &vortex_session::VortexSession) -> Self
+
 impl vortex_array::dtype::arrow::FromArrowType<&arrow_schema::field::Field> for vortex_array::dtype::DType
 
 pub fn vortex_array::dtype::DType::from_arrow(field: &arrow_schema::field::Field) -> Self
 
+pub fn vortex_array::dtype::DType::from_arrow_with_session(field: &arrow_schema::field::Field, session: &vortex_session::VortexSession) -> Self
+
 impl vortex_array::dtype::arrow::FromArrowType<&arrow_schema::fields::Fields> for vortex_array::dtype::StructFields
 
 pub fn vortex_array::dtype::StructFields::from_arrow(value: &arrow_schema::fields::Fields) -> Self
 
+pub fn vortex_array::dtype::StructFields::from_arrow_with_session(value: &arrow_schema::fields::Fields, session: &vortex_session::VortexSession) -> Self
+
 impl vortex_array::dtype::arrow::FromArrowType<&arrow_schema::schema::Schema> for vortex_array::dtype::DType
 
 pub fn vortex_array::dtype::DType::from_arrow(value: &arrow_schema::schema::Schema) -> Self
 
+pub fn vortex_array::dtype::DType::from_arrow_with_session(value: &arrow_schema::schema::Schema, session: &vortex_session::VortexSession) -> Self
+
 impl vortex_array::dtype::arrow::FromArrowType<(&arrow_schema::datatype::DataType, vortex_array::dtype::Nullability)> for vortex_array::dtype::DType
 
 pub fn vortex_array::dtype::DType::from_arrow((data_type, nullability): (&arrow_schema::datatype::DataType, vortex_array::dtype::Nullability)) -> Self
 
+pub fn vortex_array::dtype::DType::from_arrow_with_session(value: T, _session: &vortex_session::VortexSession) -> Self
+
 impl vortex_array::dtype::arrow::FromArrowType<alloc::sync::Arc<arrow_schema::schema::Schema>> for vortex_array::dtype::DType
 
 pub fn vortex_array::dtype::DType::from_arrow(value: arrow_schema::schema::SchemaRef) -> Self
 
+pub fn vortex_array::dtype::DType::from_arrow_with_session(value: arrow_schema::schema::SchemaRef, session: &vortex_session::VortexSession) -> Self
+
 pub trait vortex_array::dtype::arrow::TryFromArrowType<T>: core::marker::Sized
 
 pub fn vortex_array::dtype::arrow::TryFromArrowType::try_from_arrow(value: T) -> vortex_error::VortexResult<Self>
@@ -8794,10 +8806,16 @@ pub struct vortex_array::dtype::session::DTypeSession
 
 impl vortex_array::dtype::session::DTypeSession
 
+pub fn vortex_array::dtype::session::DTypeSession::arrow_canonical_for(&self, vortex_id: &vortex_array::dtype::extension::ExtId) -> core::option::Option<vortex_array::dtype::extension::ExtId>
+
 pub fn vortex_array::dtype::session::DTypeSession::register<V: vortex_array::dtype::extension::ExtVTable>(&self, vtable: V)
 
+pub fn vortex_array::dtype::session::DTypeSession::register_arrow_canonical(&self, vortex_id: vortex_array::dtype::extension::ExtId, arrow_name: &'static str)
+
 pub fn vortex_array::dtype::session::DTypeSession::registry(&self) -> &vortex_array::dtype::session::ExtDTypeRegistry
 
+pub fn vortex_array::dtype::session::DTypeSession::vortex_id_for_arrow_canonical(&self, arrow_name: &str) -> core::option::Option<vortex_array::dtype::extension::ExtId>
+
 impl core::default::Default for vortex_array::dtype::session::DTypeSession
 
 pub fn vortex_array::dtype::session::DTypeSession::default() -> Self
@@ -8964,6 +8982,8 @@ pub fn vortex_array::dtype::DType::to_arrow_dtype(&self) -> vortex_error::Vortex
 
 pub fn vortex_array::dtype::DType::to_arrow_schema(&self) -> vortex_error::VortexResult<arrow_schema::schema::Schema>
 
+pub fn vortex_array::dtype::DType::to_arrow_schema_with_session(&self, session: &vortex_session::VortexSession) -> vortex_error::VortexResult<arrow_schema::schema::Schema>
+
 impl core::clone::Clone for vortex_array::dtype::DType
 
 pub fn vortex_array::dtype::DType::clone(&self) -> vortex_array::dtype::DType
@@ -9026,18 +9046,26 @@ impl vortex_array::dtype::arrow::FromArrowType<&arrow_schema::field::Field> for
 
 pub fn vortex_array::dtype::DType::from_arrow(field: &arrow_schema::field::Field) -> Self
 
+pub fn vortex_array::dtype::DType::from_arrow_with_session(field: &arrow_schema::field::Field, session: &vortex_session::VortexSession) -> Self
+
 impl vortex_array::dtype::arrow::FromArrowType<&arrow_schema::schema::Schema> for vortex_array::dtype::DType
 
 pub fn vortex_array::dtype::DType::from_arrow(value: &arrow_schema::schema::Schema) -> Self
 
+pub fn vortex_array::dtype::DType::from_arrow_with_session(value: &arrow_schema::schema::Schema, session: &vortex_session::VortexSession) -> Self
+
 impl vortex_array::dtype::arrow::FromArrowType<(&arrow_schema::datatype::DataType, vortex_array::dtype::Nullability)> for vortex_array::dtype::DType
 
 pub fn vortex_array::dtype::DType::from_arrow((data_type, nullability): (&arrow_schema::datatype::DataType, vortex_array::dtype::Nullability)) -> Self
 
+pub fn vortex_array::dtype::DType::from_arrow_with_session(value: T, _session: &vortex_session::VortexSession) -> Self
+
 impl vortex_array::dtype::arrow::FromArrowType<alloc::sync::Arc<arrow_schema::schema::Schema>> for vortex_array::dtype::DType
 
 pub fn vortex_array::dtype::DType::from_arrow(value: arrow_schema::schema::SchemaRef) -> Self
 
+pub fn vortex_array::dtype::DType::from_arrow_with_session(value: arrow_schema::schema::SchemaRef, session: &vortex_session::VortexSession) -> Self
+
 impl vortex_flatbuffers::FlatBufferRoot for vortex_array::dtype::DType
 
 impl vortex_flatbuffers::WriteFlatBuffer for vortex_array::dtype::DType
@@ -9992,6 +10020,8 @@ impl vortex_array::dtype::arrow::FromArrowType<&arrow_schema::fields::Fields> fo
 
 pub fn vortex_array::dtype::StructFields::from_arrow(value: &arrow_schema::fields::Fields) -> Self
 
+pub fn vortex_array::dtype::StructFields::from_arrow_with_session(value: &arrow_schema::fields::Fields, session: &vortex_session::VortexSession) -> Self
+
 impl<T, V> core::iter::traits::collect::FromIterator<(T, V)> for vortex_array::dtype::StructFields where T: core::convert::Into<vortex_array::dtype::FieldName>, V: core::convert::Into<vortex_array::dtype::FieldDType>
 
 pub fn vortex_array::dtype::StructFields::from_iter<I: core::iter::traits::collect::IntoIterator<Item = (T, V)>>(iter: I) -> Self