diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 19f6c26968..4aeddb665a 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -2729,7 +2729,7 @@ pub fn iceberg::spec::TableMetadata::sort_order_by_id(&self, sort_order_id: i64) pub fn iceberg::spec::TableMetadata::sort_orders_iter(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator pub fn iceberg::spec::TableMetadata::statistics_for_snapshot(&self, snapshot_id: i64) -> core::option::Option<&iceberg::spec::StatisticsFile> pub fn iceberg::spec::TableMetadata::statistics_iter(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator -pub fn iceberg::spec::TableMetadata::table_properties(&self) -> iceberg::Result +pub fn iceberg::spec::TableMetadata::table_properties(&self) -> iceberg::spec::TableProperties<'_> pub fn iceberg::spec::TableMetadata::uuid(&self) -> uuid::Uuid pub async fn iceberg::spec::TableMetadata::write_to(&self, file_io: &iceberg::io::FileIO, metadata_location: &iceberg::MetadataLocation) -> iceberg::Result<()> impl core::clone::Clone for iceberg::spec::TableMetadata @@ -2796,111 +2796,108 @@ impl core::clone::Clone for iceberg::spec::TableMetadataBuilder pub fn iceberg::spec::TableMetadataBuilder::clone(&self) -> iceberg::spec::TableMetadataBuilder impl core::fmt::Debug for iceberg::spec::TableMetadataBuilder pub fn iceberg::spec::TableMetadataBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub struct iceberg::spec::TableProperties -impl iceberg::spec::TableProperties -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &str -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &str -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_NUM_RETRIES: &str -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &str -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 -pub const iceberg::spec::TableProperties::PROPERTY_CURRENT_SCHEMA: &str -pub const iceberg::spec::TableProperties::PROPERTY_CURRENT_SNAPSHOT_ID: &str -pub const iceberg::spec::TableProperties::PROPERTY_CURRENT_SNAPSHOT_SUMMARY: &str -pub const iceberg::spec::TableProperties::PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP: &str -pub const iceberg::spec::TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_FILE_FORMAT: &str -pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_PARTITION_SPEC: &str -pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_SORT_ORDER: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &str -pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &str -pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_KEY_ID: &str -pub const iceberg::spec::TableProperties::PROPERTY_FORMAT_VERSION: &str -pub const iceberg::spec::TableProperties::PROPERTY_GC_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_GC_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_MAX_REF_AGE_MS: &str -pub const iceberg::spec::TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT: i64 -pub const iceberg::spec::TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS: &str -pub const iceberg::spec::TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT: i64 -pub const iceberg::spec::TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC: &str -pub const iceberg::spec::TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &str -pub const iceberg::spec::TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &str -pub const iceberg::spec::TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_COMPRESSION_LEVEL: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_SNAPSHOT_COUNT: &str -pub const iceberg::spec::TableProperties::PROPERTY_UUID: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_DATA_LOCATION: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_METADATA_PATH: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT: u64 -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize -pub const iceberg::spec::TableProperties::RESERVED_PROPERTIES: [&str; 9] -pub fn iceberg::spec::TableProperties::data_encryption_key_size(&self) -> iceberg::Result -impl iceberg::spec::TableProperties -pub fn iceberg::spec::TableProperties::cdc_enabled(&self) -> bool -pub fn iceberg::spec::TableProperties::cdc_max_chunk_size(&self) -> usize -pub fn iceberg::spec::TableProperties::cdc_min_chunk_size(&self) -> usize -pub fn iceberg::spec::TableProperties::cdc_norm_level(&self) -> i32 -pub fn iceberg::spec::TableProperties::commit_max_retry_wait_ms(&self) -> u64 -pub fn iceberg::spec::TableProperties::commit_min_retry_wait_ms(&self) -> u64 -pub fn iceberg::spec::TableProperties::commit_num_retries(&self) -> usize -pub fn iceberg::spec::TableProperties::commit_total_retry_timeout_ms(&self) -> u64 -pub fn iceberg::spec::TableProperties::encryption_data_key_length(&self) -> usize -pub fn iceberg::spec::TableProperties::encryption_key_id(&self) -> &core::option::Option -pub fn iceberg::spec::TableProperties::from_properties(properties: &std::collections::hash::map::HashMap) -> iceberg::Result -pub fn iceberg::spec::TableProperties::gc_enabled(&self) -> bool -pub fn iceberg::spec::TableProperties::max_ref_age_ms(&self) -> i64 -pub fn iceberg::spec::TableProperties::max_snapshot_age_ms(&self) -> i64 -pub fn iceberg::spec::TableProperties::metadata_compression_codec(&self) -> &iceberg::compression::CompressionCodec -pub fn iceberg::spec::TableProperties::min_snapshots_to_keep(&self) -> usize -pub fn iceberg::spec::TableProperties::parquet_compression_codec(&self) -> &iceberg::compression::CompressionCodec -pub fn iceberg::spec::TableProperties::parquet_dict_size_bytes(&self) -> usize -pub fn iceberg::spec::TableProperties::parquet_page_row_limit(&self) -> usize -pub fn iceberg::spec::TableProperties::parquet_page_size_bytes(&self) -> usize -pub fn iceberg::spec::TableProperties::parquet_row_group_size_bytes(&self) -> usize -pub fn iceberg::spec::TableProperties::write_data_location(&self) -> &core::option::Option -pub fn iceberg::spec::TableProperties::write_datafusion_fanout_enabled(&self) -> bool -pub fn iceberg::spec::TableProperties::write_folder_storage_location(&self) -> &core::option::Option -pub fn iceberg::spec::TableProperties::write_format_default(&self) -> &alloc::string::String -pub fn iceberg::spec::TableProperties::write_metadata_path(&self) -> &core::option::Option -pub fn iceberg::spec::TableProperties::write_object_storage_location(&self) -> &core::option::Option -pub fn iceberg::spec::TableProperties::write_object_storage_partitioned_paths(&self) -> bool -pub fn iceberg::spec::TableProperties::write_target_file_size_bytes(&self) -> usize -impl core::convert::TryFrom<&std::collections::hash::map::HashMap> for iceberg::spec::TableProperties -pub type iceberg::spec::TableProperties::Error = iceberg::Error -pub fn iceberg::spec::TableProperties::try_from(properties: &std::collections::hash::map::HashMap) -> iceberg::Result -impl core::fmt::Debug for iceberg::spec::TableProperties -pub fn iceberg::spec::TableProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::spec::TableProperties<'properties> +impl iceberg::spec::TableProperties<'_> +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_NUM_RETRIES: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 +pub const iceberg::spec::TableProperties<'_>::PROPERTY_CURRENT_SCHEMA: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_CURRENT_SNAPSHOT_ID: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_CURRENT_SNAPSHOT_SUMMARY: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties<'_>::PROPERTY_DEFAULT_FILE_FORMAT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_DEFAULT_PARTITION_SPEC: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_DEFAULT_SORT_ORDER: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_ENCRYPTION_KEY_ID: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_FORMAT_VERSION: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_GC_ENABLED: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_GC_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties<'_>::PROPERTY_MAX_REF_AGE_MS: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_MAX_REF_AGE_MS_DEFAULT: i64 +pub const iceberg::spec::TableProperties<'_>::PROPERTY_MAX_SNAPSHOT_AGE_MS: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT: i64 +pub const iceberg::spec::TableProperties<'_>::PROPERTY_METADATA_COMPRESSION_CODEC: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_ENABLED: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_NORM_LEVEL: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_COMPRESSION_CODEC: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_COMPRESSION_CODEC_DEFAULT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_COMPRESSION_LEVEL: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_DICT_SIZE_BYTES: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_PAGE_ROW_LIMIT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_PAGE_SIZE_BYTES: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_SNAPSHOT_COUNT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_UUID: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_DATA_LOCATION: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_METADATA_PATH: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT: bool +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT: u64 +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::RESERVED_PROPERTIES: [&'static str; 9] +pub fn iceberg::spec::TableProperties<'_>::data_encryption_key_size(&self) -> iceberg::Result +impl<'properties> iceberg::spec::TableProperties<'properties> +pub fn iceberg::spec::TableProperties<'properties>::cdc_enabled(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::cdc_max_chunk_size(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::cdc_min_chunk_size(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::cdc_norm_level(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::commit_max_retry_wait_ms(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::commit_min_retry_wait_ms(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::commit_num_retries(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::commit_total_retry_timeout_ms(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::encryption_data_key_length(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::encryption_key_id(&self) -> iceberg::Result> +pub fn iceberg::spec::TableProperties<'properties>::gc_enabled(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::max_ref_age_ms(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::max_snapshot_age_ms(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::metadata_compression_codec(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::min_snapshots_to_keep(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::new(properties: &'properties std::collections::hash::map::HashMap) -> Self +pub fn iceberg::spec::TableProperties<'properties>::parquet_compression_codec(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::parquet_dict_size_bytes(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::parquet_page_row_limit(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::parquet_page_size_bytes(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::parquet_row_group_size_bytes(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::write_data_location(&self) -> iceberg::Result> +pub fn iceberg::spec::TableProperties<'properties>::write_datafusion_fanout_enabled(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::write_folder_storage_location(&self) -> iceberg::Result> +pub fn iceberg::spec::TableProperties<'properties>::write_format_default(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::write_metadata_path(&self) -> iceberg::Result> +pub fn iceberg::spec::TableProperties<'properties>::write_object_storage_location(&self) -> iceberg::Result> +pub fn iceberg::spec::TableProperties<'properties>::write_object_storage_partitioned_paths(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::write_target_file_size_bytes(&self) -> iceberg::Result +impl<'properties> core::fmt::Debug for iceberg::spec::TableProperties<'properties> +pub fn iceberg::spec::TableProperties<'properties>::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub struct iceberg::spec::UnboundPartitionField pub iceberg::spec::UnboundPartitionField::field_id: core::option::Option pub iceberg::spec::UnboundPartitionField::name: alloc::string::String @@ -3407,7 +3404,7 @@ pub async fn iceberg::writer::file_writer::ParquetWriter::close(self) -> iceberg pub async fn iceberg::writer::file_writer::ParquetWriter::write(&mut self, batch: &arrow_array::record_batch::RecordBatch) -> iceberg::Result<()> pub struct iceberg::writer::file_writer::ParquetWriterBuilder impl iceberg::writer::file_writer::ParquetWriterBuilder -pub fn iceberg::writer::file_writer::ParquetWriterBuilder::from_table_properties(table_props: &iceberg::spec::TableProperties, schema: iceberg::spec::SchemaRef) -> iceberg::Result +pub fn iceberg::writer::file_writer::ParquetWriterBuilder::from_table_properties(table_props: &iceberg::spec::TableProperties<'_>, schema: iceberg::spec::SchemaRef) -> iceberg::Result pub fn iceberg::writer::file_writer::ParquetWriterBuilder::new(props: parquet::file::properties::WriterProperties, schema: iceberg::spec::SchemaRef) -> Self pub fn iceberg::writer::file_writer::ParquetWriterBuilder::new_with_match_mode(props: parquet::file::properties::WriterProperties, schema: iceberg::spec::SchemaRef, match_mode: iceberg::arrow::FieldMatchMode) -> Self pub fn iceberg::writer::file_writer::ParquetWriterBuilder::with_encryption_manager(self, encryption_manager: alloc::sync::Arc) -> Self diff --git a/crates/iceberg/src/catalog/metadata_location.rs b/crates/iceberg/src/catalog/metadata_location.rs index d5daa02b37..8240a326f3 100644 --- a/crates/iceberg/src/catalog/metadata_location.rs +++ b/crates/iceberg/src/catalog/metadata_location.rs @@ -15,14 +15,13 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Display; use std::str::FromStr; use uuid::Uuid; use crate::compression::CompressionCodec; -use crate::spec::{TableMetadata, parse_metadata_file_compression}; +use crate::spec::TableMetadata; use crate::{Error, ErrorKind, Result}; /// Default folder name for metadata files under the table location, used when the @@ -43,12 +42,6 @@ pub struct MetadataLocation { } impl MetadataLocation { - /// Determines the compression codec from table properties. - /// Parse errors result in CompressionCodec::None. - fn compression_from_properties(properties: &HashMap) -> CompressionCodec { - parse_metadata_file_compression(properties).unwrap_or(CompressionCodec::None) - } - /// Creates a completely new metadata location starting at version 0, deriving the /// metadata directory and compression settings from the table metadata. /// Only used for creating a new table. For updates, see `with_next_version` and @@ -58,7 +51,7 @@ impl MetadataLocation { location: metadata.metadata_location()?, version: 0, id: Uuid::new_v4(), - compression_codec: Self::compression_from_properties(metadata.properties()), + compression_codec: metadata.metadata_compression_codec()?, }) } @@ -80,7 +73,7 @@ impl MetadataLocation { location: new_metadata.metadata_location()?, version: self.version, id: self.id, - compression_codec: Self::compression_from_properties(new_metadata.properties()), + compression_codec: new_metadata.metadata_compression_codec()?, }) } diff --git a/crates/iceberg/src/catalog/utils.rs b/crates/iceberg/src/catalog/utils.rs index 1611db8302..6e6997b214 100644 --- a/crates/iceberg/src/catalog/utils.rs +++ b/crates/iceberg/src/catalog/utils.rs @@ -61,7 +61,7 @@ pub async fn drop_table_data(table_info: &Table) -> Result<()> { } // Delete data files only if gc.enabled is true, to avoid corrupting shared tables - if metadata.table_properties()?.gc_enabled() { + if metadata.table_properties().gc_enabled()? { delete_data_files(io, &manifests_to_delete).await?; } diff --git a/crates/iceberg/src/encryption/manager.rs b/crates/iceberg/src/encryption/manager.rs index 63e6438bd5..f54ef464d5 100644 --- a/crates/iceberg/src/encryption/manager.rs +++ b/crates/iceberg/src/encryption/manager.rs @@ -113,8 +113,9 @@ impl EncryptionManager { return Ok(None); } - let table_properties = metadata.table_properties()?; - let Some(table_key_id) = table_properties.encryption_key_id().as_deref() else { + let table_properties = metadata.table_properties(); + let encryption_key_id = table_properties.encryption_key_id()?; + let Some(table_key_id) = encryption_key_id.as_deref() else { if kms_client.is_some() { tracing::warn!( "KeyManagementClient provided but table does not have encryption.key-id set" diff --git a/crates/iceberg/src/spec/mod.rs b/crates/iceberg/src/spec/mod.rs index b23ca1eda0..707ebbb630 100644 --- a/crates/iceberg/src/spec/mod.rs +++ b/crates/iceberg/src/spec/mod.rs @@ -50,7 +50,6 @@ pub use sort::*; pub use statistic_file::*; pub use table_metadata::*; pub(crate) use table_metadata_builder::FIRST_FIELD_ID; -pub(crate) use table_properties::parse_metadata_file_compression; pub use table_properties::*; pub use transform::*; pub(crate) use values::decimal_utils; diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index bb455a4a5c..92bb6ff4d8 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -35,7 +35,7 @@ pub use super::table_metadata_builder::{TableMetadataBuildResult, TableMetadataB use super::{ DEFAULT_PARTITION_SPEC_ID, PartitionSpecRef, PartitionStatisticsFile, SchemaId, SchemaRef, SnapshotRef, SnapshotRetention, SortOrder, SortOrderRef, StatisticsFile, StructType, - TableProperties, parse_metadata_file_compression, + TableProperties, }; use crate::catalog::{METADATA_FOLDER_NAME, MetadataLocation}; use crate::compression::CompressionCodec; @@ -370,9 +370,8 @@ impl TableMetadata { /// to the `metadata` subdirectory under the table location. pub fn metadata_location(&self) -> Result { Ok(self - .table_properties()? - .write_metadata_path() - .clone() + .table_properties() + .write_metadata_path()? .unwrap_or_else(|| format!("{}/{}", self.location(), METADATA_FOLDER_NAME))) } @@ -385,14 +384,13 @@ impl TableMetadata { /// /// Returns an error if the compression codec property has an invalid value. pub fn metadata_compression_codec(&self) -> Result { - parse_metadata_file_compression(&self.properties) + self.table_properties().metadata_compression_codec() } - /// Returns typed table properties parsed from the raw properties map with defaults. - pub fn table_properties(&self) -> Result { - TableProperties::try_from(&self.properties).map_err(|e| { - Error::new(ErrorKind::DataInvalid, "Invalid table properties").with_source(e) - }) + /// Returns a typed view that parses each table property when its getter is called. + #[inline] + pub fn table_properties(&self) -> TableProperties<'_> { + TableProperties::new(&self.properties) } /// Return location of statistics files. @@ -499,7 +497,7 @@ impl TableMetadata { let json_data = serde_json::to_vec(self)?; // Check if compression codec from properties matches the one in metadata_location - let codec = parse_metadata_file_compression(&self.properties)?; + let codec = self.table_properties().metadata_compression_codec()?; if codec != metadata_location.compression_codec() { return Err(Error::new( @@ -4041,14 +4039,14 @@ mod tests { .unwrap() .metadata; - let props = metadata.table_properties().unwrap(); + let props = metadata.table_properties(); assert_eq!( - props.commit_num_retries(), + props.commit_num_retries().unwrap(), TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT ); assert_eq!( - props.write_target_file_size_bytes(), + props.write_target_file_size_bytes().unwrap(), TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT ); } @@ -4088,10 +4086,67 @@ mod tests { .unwrap() .metadata; - let props = metadata.table_properties().unwrap(); + let props = metadata.table_properties(); + + assert_eq!(props.commit_num_retries().unwrap(), 10); + assert_eq!(props.write_target_file_size_bytes().unwrap(), 1024); + } + + #[test] + fn test_deserialize_metadata_defers_invalid_table_property_errors() { + let invalid_retries = "not_a_number"; + let invalid_codec = "unknown"; + let target_file_size = "1024"; + + for file_name in [ + "TableMetadataV1Valid.json", + "TableMetadataV2ValidMinimal.json", + "TableMetadataV3ValidMinimal.json", + ] { + let path = format!("testdata/table_metadata/{file_name}"); + let mut json: serde_json::Value = + serde_json::from_str(&fs::read_to_string(path).unwrap()).unwrap(); + json["properties"] = serde_json::json!({ + (TableProperties::PROPERTY_COMMIT_NUM_RETRIES): invalid_retries, + (TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC): invalid_codec, + (TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES): target_file_size, + }); + + let metadata: TableMetadata = serde_json::from_value(json).unwrap(); + assert_eq!( + metadata + .properties() + .get(TableProperties::PROPERTY_COMMIT_NUM_RETRIES) + .map(String::as_str), + Some(invalid_retries) + ); + + let table_properties = metadata.table_properties(); + let error = table_properties.commit_num_retries().unwrap_err(); + assert!( + error + .message() + .contains(TableProperties::PROPERTY_COMMIT_NUM_RETRIES) + ); + assert_eq!( + table_properties.write_target_file_size_bytes().unwrap(), + 1024 + ); + let error = table_properties.metadata_compression_codec().unwrap_err(); + assert!( + format!("{error}").contains(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC) + ); - assert_eq!(props.commit_num_retries(), 10); - assert_eq!(props.write_target_file_size_bytes(), 1024); + let serialized = serde_json::to_value(metadata).unwrap(); + assert_eq!( + serialized["properties"][TableProperties::PROPERTY_COMMIT_NUM_RETRIES], + invalid_retries + ); + assert_eq!( + serialized["properties"][TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC], + invalid_codec + ); + } } #[test] @@ -4103,10 +4158,16 @@ mod tests { .build() .unwrap(); - let properties = HashMap::from([( - "commit.retry.num-retries".to_string(), - "not_a_number".to_string(), - )]); + let properties = HashMap::from([ + ( + TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(), + "not_a_number".to_string(), + ), + ( + TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(), + "1024".to_string(), + ), + ]); let metadata = TableMetadataBuilder::new( schema, @@ -4121,9 +4182,17 @@ mod tests { .unwrap() .metadata; - let err = metadata.table_properties().unwrap_err(); + let table_properties = metadata.table_properties(); + let err = table_properties.commit_num_retries().unwrap_err(); assert_eq!(err.kind(), ErrorKind::DataInvalid); - assert!(err.message().contains("Invalid table properties")); + assert!( + err.message() + .contains(TableProperties::PROPERTY_COMMIT_NUM_RETRIES) + ); + assert_eq!( + table_properties.write_target_file_size_bytes().unwrap(), + 1024 + ); } #[test] diff --git a/crates/iceberg/src/spec/table_metadata_builder.rs b/crates/iceberg/src/spec/table_metadata_builder.rs index 3191d6c13c..6467014ab8 100644 --- a/crates/iceberg/src/spec/table_metadata_builder.rs +++ b/crates/iceberg/src/spec/table_metadata_builder.rs @@ -2095,6 +2095,32 @@ mod tests { assert_eq!(build_result.metadata.metadata_log.len(), 0); } + #[test] + fn test_table_properties_view_reflects_metadata_updates() { + let property = TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(); + let metadata = builder_without_changes(FormatVersion::V2) + .set_properties(HashMap::from([(property.clone(), "7".to_string())])) + .unwrap() + .build() + .unwrap() + .metadata; + + assert_eq!(metadata.table_properties().commit_num_retries().unwrap(), 7); + + let metadata = metadata + .into_builder(None) + .remove_properties(&[property]) + .unwrap() + .build() + .unwrap() + .metadata; + + assert_eq!( + metadata.table_properties().commit_num_retries().unwrap(), + TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT + ); + } + #[test] fn test_no_metadata_log_entry_for_no_previous_location() { // Used for first commit after stage-creation of tables diff --git a/crates/iceberg/src/spec/table_properties.rs b/crates/iceberg/src/spec/table_properties.rs index 7ed1c032d5..b927d36386 100644 --- a/crates/iceberg/src/spec/table_properties.rs +++ b/crates/iceberg/src/spec/table_properties.rs @@ -17,7 +17,7 @@ use std::collections::HashMap; -use iceberg_property_macro::Properties; +use iceberg_property_macro::properties_view; use crate::compression::CompressionCodec; use crate::encryption::AesKeySize; @@ -32,29 +32,6 @@ fn parse_location_property(path: &str) -> Result { Ok(strip_trailing_slash(path).to_string()) } -/// Parse compression codec for metadata files from table properties. -/// Retrieves the compression codec property, applies defaults, and parses the value. -/// Only "none" (or empty string) and "gzip" are supported for metadata compression. -/// -/// # Arguments -/// -/// * `properties` - HashMap containing table properties -/// -/// # Errors -/// -/// Returns an error if the codec is not "none", "", or "gzip" (case-insensitive). -/// Lz4 and Zstd are not supported for metadata file compression. -pub(crate) fn parse_metadata_file_compression( - properties: &HashMap, -) -> Result { - let value = properties - .get(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC) - .map(|s| s.as_str()) - .unwrap_or(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT); - - parse_metadata_compression(value) -} - fn parse_metadata_compression(value: &str) -> Result { // Handle empty string as None if value.is_empty() { @@ -139,8 +116,9 @@ fn parse_parquet_compression( }) } -/// TableProperties that contains the properties of a table. -#[derive(Debug, Properties)] +properties_view! { +/// Typed view over the properties of a table. +#[derive(Debug)] pub struct TableProperties { /// The number of times to retry a commit. #[property( @@ -359,8 +337,9 @@ pub struct TableProperties { )] write_object_storage_partitioned_paths: bool, } +} -impl TableProperties { +impl TableProperties<'_> { /// Reserved table property for table format version. /// /// Iceberg will default a new table's format version to the latest stable and recommended @@ -370,32 +349,33 @@ impl TableProperties { /// If this table property exists when creating a table, the table will use the specified format /// version. If a table updates this property, it will try to upgrade to the specified format /// version. - pub const PROPERTY_FORMAT_VERSION: &str = "format-version"; + pub const PROPERTY_FORMAT_VERSION: &'static str = "format-version"; /// Reserved table property for table UUID. - pub const PROPERTY_UUID: &str = "uuid"; + pub const PROPERTY_UUID: &'static str = "uuid"; /// Reserved table property for the total number of snapshots. - pub const PROPERTY_SNAPSHOT_COUNT: &str = "snapshot-count"; + pub const PROPERTY_SNAPSHOT_COUNT: &'static str = "snapshot-count"; /// Reserved table property for current snapshot summary. - pub const PROPERTY_CURRENT_SNAPSHOT_SUMMARY: &str = "current-snapshot-summary"; + pub const PROPERTY_CURRENT_SNAPSHOT_SUMMARY: &'static str = "current-snapshot-summary"; /// Reserved table property for current snapshot id. - pub const PROPERTY_CURRENT_SNAPSHOT_ID: &str = "current-snapshot-id"; + pub const PROPERTY_CURRENT_SNAPSHOT_ID: &'static str = "current-snapshot-id"; /// Reserved table property for current snapshot timestamp. - pub const PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP: &str = "current-snapshot-timestamp-ms"; + pub const PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP: &'static str = "current-snapshot-timestamp-ms"; /// Reserved table property for the JSON representation of current schema. - pub const PROPERTY_CURRENT_SCHEMA: &str = "current-schema"; + pub const PROPERTY_CURRENT_SCHEMA: &'static str = "current-schema"; /// Reserved table property for the JSON representation of current(default) partition spec. - pub const PROPERTY_DEFAULT_PARTITION_SPEC: &str = "default-partition-spec"; + pub const PROPERTY_DEFAULT_PARTITION_SPEC: &'static str = "default-partition-spec"; /// Reserved table property for the JSON representation of current(default) sort order. - pub const PROPERTY_DEFAULT_SORT_ORDER: &str = "default-sort-order"; + pub const PROPERTY_DEFAULT_SORT_ORDER: &'static str = "default-sort-order"; /// Property key for max number of previous versions to keep. - pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &str = + pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &'static str = "write.metadata.previous-versions-max"; /// Default value for max number of previous versions to keep. pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT: usize = 100; /// Property key for max number of partitions to keep summary stats for. - pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &str = "write.summary.partition-limit"; + pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &'static str = + "write.summary.partition-limit"; /// Default value for the max number of partitions to keep summary stats for. pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT: u64 = 0; @@ -403,7 +383,7 @@ impl TableProperties { /// /// Reserved table properties are only used to control behaviors when creating or updating a /// table. The value of these properties are not persisted as a part of the table metadata. - pub const RESERVED_PROPERTIES: [&str; 9] = [ + pub const RESERVED_PROPERTIES: [&'static str; 9] = [ Self::PROPERTY_FORMAT_VERSION, Self::PROPERTY_UUID, Self::PROPERTY_SNAPSHOT_COUNT, @@ -416,88 +396,91 @@ impl TableProperties { ]; /// Property key for number of commit retries. - pub const PROPERTY_COMMIT_NUM_RETRIES: &str = "commit.retry.num-retries"; + pub const PROPERTY_COMMIT_NUM_RETRIES: &'static str = "commit.retry.num-retries"; /// Default value for number of commit retries. pub const PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize = 4; /// Property key for minimum wait time (ms) between retries. - pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &str = "commit.retry.min-wait-ms"; + pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &'static str = "commit.retry.min-wait-ms"; /// Default value for minimum wait time (ms) between retries. pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 = 100; /// Property key for maximum wait time (ms) between retries. - pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &str = "commit.retry.max-wait-ms"; + pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &'static str = "commit.retry.max-wait-ms"; /// Default value for maximum wait time (ms) between retries. pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 = 60 * 1000; // 1 minute /// Property key for total maximum retry time (ms). - pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &str = "commit.retry.total-timeout-ms"; + pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &'static str = "commit.retry.total-timeout-ms"; /// Default value for total maximum retry time (ms). pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 = 30 * 60 * 1000; // 30 minutes /// Default file format for data files - pub const PROPERTY_DEFAULT_FILE_FORMAT: &str = "write.format.default"; + pub const PROPERTY_DEFAULT_FILE_FORMAT: &'static str = "write.format.default"; /// Default file format for delete files - pub const PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &str = "write.delete.format.default"; + pub const PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &'static str = "write.delete.format.default"; /// Default value for data file format - pub const PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &str = "parquet"; + pub const PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &'static str = "parquet"; /// Target file size for newly written files. - pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str = "write.target-file-size-bytes"; + pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &'static str = "write.target-file-size-bytes"; /// Default target file size pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize = 512 * 1024 * 1024; // 512 MB /// Base location for metadata files (manifests, manifest lists, table metadata). /// When unset, metadata files default to the `metadata` directory under the table /// location. - pub const PROPERTY_WRITE_METADATA_PATH: &str = "write.metadata.path"; + pub const PROPERTY_WRITE_METADATA_PATH: &'static str = "write.metadata.path"; /// Compression codec for metadata files (JSON) - pub const PROPERTY_METADATA_COMPRESSION_CODEC: &str = "write.metadata.compression-codec"; + pub const PROPERTY_METADATA_COMPRESSION_CODEC: &'static str = + "write.metadata.compression-codec"; /// Default metadata compression codec - uncompressed - pub const PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &str = "none"; + pub const PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &'static str = "none"; /// Whether to use `FanoutWriter` for partitioned tables (handles unsorted data). /// If false, uses `ClusteredWriter` (requires sorted data, more memory efficient). - pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &str = "write.datafusion.fanout.enabled"; + pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &'static str = + "write.datafusion.fanout.enabled"; /// Default value for fanout writer enabled pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool = true; /// Property key for enabling garbage collection on drop. /// When set to `false`, data files will not be deleted when a table is dropped. /// Defaults to `true`. - pub const PROPERTY_GC_ENABLED: &str = "gc.enabled"; + pub const PROPERTY_GC_ENABLED: &'static str = "gc.enabled"; /// Default value for gc.enabled pub const PROPERTY_GC_ENABLED_DEFAULT: bool = true; /// Property key for the default maximum age of a snapshot to keep when expiring snapshots. - pub const PROPERTY_MAX_SNAPSHOT_AGE_MS: &str = "history.expire.max-snapshot-age-ms"; + pub const PROPERTY_MAX_SNAPSHOT_AGE_MS: &'static str = "history.expire.max-snapshot-age-ms"; /// Default value for history.expire.max-snapshot-age-ms (5 days). pub const PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT: i64 = 5 * 24 * 60 * 60 * 1000; /// Property key for the default minimum number of snapshots to keep when expiring snapshots. - pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &str = "history.expire.min-snapshots-to-keep"; + pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &'static str = "history.expire.min-snapshots-to-keep"; /// Default value for history.expire.min-snapshots-to-keep. pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize = 1; /// Property key for the default maximum age of a snapshot reference to keep when expiring. - pub const PROPERTY_MAX_REF_AGE_MS: &str = "history.expire.max-ref-age-ms"; + pub const PROPERTY_MAX_REF_AGE_MS: &'static str = "history.expire.max-ref-age-ms"; /// Default value for history.expire.max-ref-age-ms (effectively never expire refs). pub const PROPERTY_MAX_REF_AGE_MS_DEFAULT: i64 = i64::MAX; /// Enable content-defined chunking with parquet defaults (or per-property overrides). - pub const PROPERTY_PARQUET_CDC_ENABLED: &str = "write.parquet.content-defined-chunking.enabled"; + pub const PROPERTY_PARQUET_CDC_ENABLED: &'static str = + "write.parquet.content-defined-chunking.enabled"; /// Default value for content-defined chunking enabled. pub const PROPERTY_PARQUET_CDC_ENABLED_DEFAULT: bool = false; /// Minimum chunk size in bytes for content-defined chunking. - pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &str = + pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &'static str = "write.parquet.content-defined-chunking.min-chunk-size"; /// Default matches `parquet::file::properties::DEFAULT_CDC_MIN_CHUNK_SIZE`. pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT: usize = 256 * 1024; /// Maximum chunk size in bytes for content-defined chunking. - pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &str = + pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &'static str = "write.parquet.content-defined-chunking.max-chunk-size"; /// Default matches `parquet::file::properties::DEFAULT_CDC_MAX_CHUNK_SIZE`. pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT: usize = 1024 * 1024; /// Normalization level (gearhash bit adjustment) for content-defined chunking. - pub const PROPERTY_PARQUET_CDC_NORM_LEVEL: &str = + pub const PROPERTY_PARQUET_CDC_NORM_LEVEL: &'static str = "write.parquet.content-defined-chunking.norm-level"; /// Default matches `parquet::file::properties::DEFAULT_CDC_NORM_LEVEL`. pub const PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 = 0; @@ -506,49 +489,50 @@ impl TableProperties { /// `lz4`, `lz4_raw`, `brotli`, `lzo`, `uncompressed`). The codec name is /// parsed into a [`CompressionCodec`] when properties are parsed; the level's /// range is validated when the writer is built. - pub const PROPERTY_PARQUET_COMPRESSION_CODEC: &str = "write.parquet.compression-codec"; + pub const PROPERTY_PARQUET_COMPRESSION_CODEC: &'static str = "write.parquet.compression-codec"; /// Default Parquet compression codec. - pub const PROPERTY_PARQUET_COMPRESSION_CODEC_DEFAULT: &str = "zstd"; + pub const PROPERTY_PARQUET_COMPRESSION_CODEC_DEFAULT: &'static str = "zstd"; /// Compression level for Parquet data files, for codecs that take one /// (`gzip`, `zstd`, `brotli`). When unset, the codec's default level is used. - pub const PROPERTY_PARQUET_COMPRESSION_LEVEL: &str = "write.parquet.compression-level"; + pub const PROPERTY_PARQUET_COMPRESSION_LEVEL: &'static str = "write.parquet.compression-level"; /// Approximate maximum size of a Parquet row group in bytes. - pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES: &str = "write.parquet.row-group-size-bytes"; + pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES: &'static str = + "write.parquet.row-group-size-bytes"; /// Default Parquet row group size in bytes. pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT: usize = 128 * 1024 * 1024; /// Approximate maximum size of a Parquet data page in bytes. - pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES: &str = "write.parquet.page-size-bytes"; + pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES: &'static str = "write.parquet.page-size-bytes"; /// Default Parquet page size in bytes. pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT: usize = 1024 * 1024; /// Maximum number of rows per Parquet data page. - pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT: &str = "write.parquet.page-row-limit"; + pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT: &'static str = "write.parquet.page-row-limit"; /// Default Parquet page row limit. pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT: usize = 20000; /// Approximate maximum size of the Parquet dictionary page in bytes. - pub const PROPERTY_PARQUET_DICT_SIZE_BYTES: &str = "write.parquet.dict-size-bytes"; + pub const PROPERTY_PARQUET_DICT_SIZE_BYTES: &'static str = "write.parquet.dict-size-bytes"; /// Default Parquet dictionary page size in bytes. pub const PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT: usize = 2 * 1024 * 1024; /// Property key for the master key id used to encrypt the table's manifest /// list and data files as defined in . - pub const PROPERTY_ENCRYPTION_KEY_ID: &str = "encryption.key-id"; + pub const PROPERTY_ENCRYPTION_KEY_ID: &'static str = "encryption.key-id"; /// Property key for the encryption data encryption key (DEK) length in bytes. - pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &str = "encryption.data-key-length"; + pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &'static str = "encryption.data-key-length"; /// Default value for the encryption DEK length (16 bytes = AES-128). pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize = 16; /// Property key for the base directory for data files - pub const PROPERTY_WRITE_DATA_LOCATION: &str = "write.data.path"; + pub const PROPERTY_WRITE_DATA_LOCATION: &'static str = "write.data.path"; /// Property key for deprecated [`TableProperties::write_folder_storage_location`] - pub const PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &str = "write.folder-storage.path"; + pub const PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &'static str = "write.folder-storage.path"; /// Property key for deprecated object storage path, kept as a fallback for compatibility. - pub const PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &str = "write.object-storage.path"; + pub const PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &'static str = "write.object-storage.path"; /// Property key for controlling whether partition values are included in object storage paths. - pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &str = + pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &'static str = "write.object-storage.partitioned-paths"; /// Default value for [`TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS`] pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT: bool = true; @@ -556,15 +540,7 @@ impl TableProperties { /// The AES key size to use when generating data encryption keys, derived /// from `encryption.data-key-length`. pub fn data_encryption_key_size(&self) -> Result { - AesKeySize::from_key_length(self.encryption_data_key_length) - } -} - -impl TryFrom<&HashMap> for TableProperties { - type Error = Error; - - fn try_from(properties: &HashMap) -> Result { - Self::from_properties(properties) + AesKeySize::from_key_length(self.encryption_data_key_length()?) } } @@ -576,46 +552,46 @@ mod tests { #[test] fn test_table_properties_default() { let props = HashMap::new(); - let table_properties = TableProperties::try_from(&props).unwrap(); + let table_properties = TableProperties::new(&props); assert_eq!( - table_properties.commit_num_retries, + table_properties.commit_num_retries().unwrap(), TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT ); assert_eq!( - table_properties.commit_min_retry_wait_ms, + table_properties.commit_min_retry_wait_ms().unwrap(), TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT ); assert_eq!( - table_properties.commit_max_retry_wait_ms, + table_properties.commit_max_retry_wait_ms().unwrap(), TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT ); assert_eq!( - table_properties.write_format_default, + table_properties.write_format_default().unwrap(), TableProperties::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT.to_string() ); assert_eq!( - table_properties.write_target_file_size_bytes, + table_properties.write_target_file_size_bytes().unwrap(), TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT ); // Test compression defaults (none means CompressionCodec::None) assert_eq!( - table_properties.metadata_compression_codec, + table_properties.metadata_compression_codec().unwrap(), CompressionCodec::None ); assert_eq!( - table_properties.gc_enabled, + table_properties.gc_enabled().unwrap(), TableProperties::PROPERTY_GC_ENABLED_DEFAULT ); assert_eq!( - table_properties.max_snapshot_age_ms, + table_properties.max_snapshot_age_ms().unwrap(), TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT ); assert_eq!( - table_properties.min_snapshots_to_keep, + table_properties.min_snapshots_to_keep().unwrap(), TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT ); assert_eq!( - table_properties.max_ref_age_ms, + table_properties.max_ref_age_ms().unwrap(), TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT ); } @@ -636,20 +612,27 @@ mod tests { "5678".to_string(), ), ]); - let table_properties = TableProperties::try_from(&props).unwrap(); - assert_eq!(table_properties.max_snapshot_age_ms, 1234); - assert_eq!(table_properties.min_snapshots_to_keep, 7); - assert_eq!(table_properties.max_ref_age_ms, 5678); + let table_properties = TableProperties::new(&props); + assert_eq!(table_properties.max_snapshot_age_ms().unwrap(), 1234); + assert_eq!(table_properties.min_snapshots_to_keep().unwrap(), 7); + assert_eq!(table_properties.max_ref_age_ms().unwrap(), 5678); } #[test] fn test_table_properties_location_paths() { // Test unset. - let table_properties = TableProperties::try_from(&HashMap::new()).unwrap(); - assert_eq!(table_properties.write_metadata_path, None); - assert_eq!(table_properties.write_data_location, None); - assert_eq!(table_properties.write_folder_storage_location, None); - assert_eq!(table_properties.write_object_storage_location, None); + let raw_properties = HashMap::new(); + let table_properties = TableProperties::new(&raw_properties); + assert_eq!(table_properties.write_metadata_path().unwrap(), None); + assert_eq!(table_properties.write_data_location().unwrap(), None); + assert_eq!( + table_properties.write_folder_storage_location().unwrap(), + None + ); + assert_eq!( + table_properties.write_object_storage_location().unwrap(), + None + ); for key in [ TableProperties::PROPERTY_WRITE_METADATA_PATH, @@ -658,34 +641,48 @@ mod tests { TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION, ] { // Test empty paths are invalid and retain the property key as error context. - let error = - TableProperties::try_from(&HashMap::from([(key.to_string(), String::new())])) - .unwrap_err(); + let raw_properties = HashMap::from([(key.to_string(), String::new())]); + let table_properties = TableProperties::new(&raw_properties); + let error = match key { + TableProperties::PROPERTY_WRITE_METADATA_PATH => { + table_properties.write_metadata_path().unwrap_err() + } + TableProperties::PROPERTY_WRITE_DATA_LOCATION => { + table_properties.write_data_location().unwrap_err() + } + TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION => table_properties + .write_folder_storage_location() + .unwrap_err(), + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION => table_properties + .write_object_storage_location() + .unwrap_err(), + _ => unreachable!(), + }; assert_eq!(error.kind(), ErrorKind::DataInvalid); assert!(format!("{error}").contains(key)); // Test all supported location properties share trailing-slash normalization. - let table_properties = TableProperties::try_from(&HashMap::from([( + let raw_properties = HashMap::from([( key.to_string(), "s3://other-bucket/custom-path/".to_string(), - )])) - .unwrap(); + )]); + let table_properties = TableProperties::new(&raw_properties); let parsed = match key { TableProperties::PROPERTY_WRITE_METADATA_PATH => { - table_properties.write_metadata_path.as_deref() + table_properties.write_metadata_path().unwrap() } TableProperties::PROPERTY_WRITE_DATA_LOCATION => { - table_properties.write_data_location.as_deref() + table_properties.write_data_location().unwrap() } TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION => { - table_properties.write_folder_storage_location.as_deref() + table_properties.write_folder_storage_location().unwrap() } TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION => { - table_properties.write_object_storage_location.as_deref() + table_properties.write_object_storage_location().unwrap() } _ => unreachable!(), }; - assert_eq!(parsed, Some("s3://other-bucket/custom-path")); + assert_eq!(parsed.as_deref(), Some("s3://other-bucket/custom-path")); } } @@ -695,9 +692,9 @@ mod tests { TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), "gzip".to_string(), )]); - let table_properties = TableProperties::try_from(&props).unwrap(); + let table_properties = TableProperties::new(&props); assert_eq!( - table_properties.metadata_compression_codec, + table_properties.metadata_compression_codec().unwrap(), CompressionCodec::gzip_default() ); } @@ -708,9 +705,9 @@ mod tests { TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), "none".to_string(), )]); - let table_properties = TableProperties::try_from(&props).unwrap(); + let table_properties = TableProperties::new(&props); assert_eq!( - table_properties.metadata_compression_codec, + table_properties.metadata_compression_codec().unwrap(), CompressionCodec::None ); } @@ -722,9 +719,9 @@ mod tests { TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), "GZIP".to_string(), )]); - let table_properties = TableProperties::try_from(&props_upper).unwrap(); + let table_properties = TableProperties::new(&props_upper); assert_eq!( - table_properties.metadata_compression_codec, + table_properties.metadata_compression_codec().unwrap(), CompressionCodec::gzip_default() ); @@ -733,9 +730,9 @@ mod tests { TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), "GzIp".to_string(), )]); - let table_properties = TableProperties::try_from(&props_mixed).unwrap(); + let table_properties = TableProperties::new(&props_mixed); assert_eq!( - table_properties.metadata_compression_codec, + table_properties.metadata_compression_codec().unwrap(), CompressionCodec::gzip_default() ); @@ -744,9 +741,9 @@ mod tests { TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), "NONE".to_string(), )]); - let table_properties = TableProperties::try_from(&props_none_upper).unwrap(); + let table_properties = TableProperties::new(&props_none_upper); assert_eq!( - table_properties.metadata_compression_codec, + table_properties.metadata_compression_codec().unwrap(), CompressionCodec::None ); } @@ -775,12 +772,18 @@ mod tests { "false".to_string(), ), ]); - let table_properties = TableProperties::try_from(&props).unwrap(); - assert_eq!(table_properties.commit_num_retries, 10); - assert_eq!(table_properties.commit_max_retry_wait_ms, 20); - assert_eq!(table_properties.write_format_default, "avro".to_string()); - assert_eq!(table_properties.write_target_file_size_bytes, 512); - assert!(!table_properties.gc_enabled); + let table_properties = TableProperties::new(&props); + assert_eq!(table_properties.commit_num_retries().unwrap(), 10); + assert_eq!(table_properties.commit_max_retry_wait_ms().unwrap(), 20); + assert_eq!( + table_properties.write_format_default().unwrap(), + "avro".to_string() + ); + assert_eq!( + table_properties.write_target_file_size_bytes().unwrap(), + 512 + ); + assert!(!table_properties.gc_enabled().unwrap()); } #[test] @@ -790,9 +793,10 @@ mod tests { "abc".to_string(), )]); - let table_properties = TableProperties::try_from(&invalid_retries).unwrap_err(); + let table_properties = TableProperties::new(&invalid_retries); + let error = table_properties.commit_num_retries().unwrap_err(); assert!( - table_properties.to_string().contains( + error.to_string().contains( "Invalid value for commit.retry.num-retries: invalid digit found in string" ) ); @@ -801,9 +805,10 @@ mod tests { TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS.to_string(), "abc".to_string(), )]); - let table_properties = TableProperties::try_from(&invalid_min_wait).unwrap_err(); + let table_properties = TableProperties::new(&invalid_min_wait); + let error = table_properties.commit_min_retry_wait_ms().unwrap_err(); assert!( - table_properties.to_string().contains( + error.to_string().contains( "Invalid value for commit.retry.min-wait-ms: invalid digit found in string" ) ); @@ -812,9 +817,10 @@ mod tests { TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS.to_string(), "abc".to_string(), )]); - let table_properties = TableProperties::try_from(&invalid_max_wait).unwrap_err(); + let table_properties = TableProperties::new(&invalid_max_wait); + let error = table_properties.commit_max_retry_wait_ms().unwrap_err(); assert!( - table_properties.to_string().contains( + error.to_string().contains( "Invalid value for commit.retry.max-wait-ms: invalid digit found in string" ) ); @@ -823,8 +829,9 @@ mod tests { TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(), "abc".to_string(), )]); - let table_properties = TableProperties::try_from(&invalid_target_size).unwrap_err(); - assert!(table_properties.to_string().contains( + let table_properties = TableProperties::new(&invalid_target_size); + let error = table_properties.write_target_file_size_bytes().unwrap_err(); + assert!(error.to_string().contains( "Invalid value for write.target-file-size-bytes: invalid digit found in string" )); @@ -832,12 +839,9 @@ mod tests { TableProperties::PROPERTY_GC_ENABLED.to_string(), "notabool".to_string(), )]); - let table_properties = TableProperties::try_from(&invalid_gc_enabled).unwrap_err(); - assert!( - table_properties - .to_string() - .contains("Invalid value for gc.enabled") - ); + let table_properties = TableProperties::new(&invalid_gc_enabled); + let error = table_properties.gc_enabled().unwrap_err(); + assert!(error.to_string().contains("Invalid value for gc.enabled")); } #[test] @@ -849,7 +853,9 @@ mod tests { TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), codec.to_string(), )]); - let err = TableProperties::try_from(&props).unwrap_err(); + let err = TableProperties::new(&props) + .metadata_compression_codec() + .unwrap_err(); let err_msg = err.to_string(); assert!( err_msg.contains(&format!("Invalid metadata compression codec: {codec}")), @@ -862,103 +868,11 @@ mod tests { } } - #[test] - fn test_parse_metadata_file_compression_valid() { - // Test with "none" - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "none".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::None - ); - - // Test with empty string - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::None - ); - - // Test with "gzip" - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "gzip".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::gzip_default() - ); - - // Test case insensitivity - "NONE" - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "NONE".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::None - ); - - // Test case insensitivity - "GZIP" - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "GZIP".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::gzip_default() - ); - - // Test case insensitivity - "GzIp" - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "GzIp".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::gzip_default() - ); - - // Test default when property is missing - let props = HashMap::new(); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::None - ); - } - - #[test] - fn test_parse_metadata_file_compression_invalid() { - let invalid_codecs = ["lz4", "zstd", "snappy"]; - - for codec in invalid_codecs { - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - codec.to_string(), - )]); - let err = parse_metadata_file_compression(&props).unwrap_err(); - let err_msg = err.to_string(); - assert!( - err_msg.contains("Invalid metadata compression codec"), - "Expected error message to contain 'Invalid metadata compression codec', got: {err_msg}" - ); - assert!( - err_msg.contains("Only 'none' and 'gzip' are supported"), - "Expected error message to contain supported codecs, got: {err_msg}" - ); - } - } - #[test] fn test_cdc_disabled_by_default() { let props = HashMap::new(); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(!tp.cdc_enabled); + let tp = TableProperties::new(&props); + assert!(!tp.cdc_enabled().unwrap()); } #[test] @@ -967,11 +881,11 @@ mod tests { TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), "true".to_string(), )]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(tp.cdc_enabled); - assert_eq!(tp.cdc_min_chunk_size, 256 * 1024); - assert_eq!(tp.cdc_max_chunk_size, 1024 * 1024); - assert_eq!(tp.cdc_norm_level, 0); + let tp = TableProperties::new(&props); + assert!(tp.cdc_enabled().unwrap()); + assert_eq!(tp.cdc_min_chunk_size().unwrap(), 256 * 1024); + assert_eq!(tp.cdc_max_chunk_size().unwrap(), 1024 * 1024); + assert_eq!(tp.cdc_norm_level().unwrap(), 0); } #[test] @@ -980,8 +894,8 @@ mod tests { TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(), "262144".to_string(), )]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(!tp.cdc_enabled); + let tp = TableProperties::new(&props); + assert!(!tp.cdc_enabled().unwrap()); } #[test] @@ -1004,11 +918,11 @@ mod tests { "1".to_string(), ), ]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(tp.cdc_enabled); - assert_eq!(tp.cdc_min_chunk_size, 200000); - assert_eq!(tp.cdc_max_chunk_size, 900000); - assert_eq!(tp.cdc_norm_level, 1); + let tp = TableProperties::new(&props); + assert!(tp.cdc_enabled().unwrap()); + assert_eq!(tp.cdc_min_chunk_size().unwrap(), 200000); + assert_eq!(tp.cdc_max_chunk_size().unwrap(), 900000); + assert_eq!(tp.cdc_norm_level().unwrap(), 1); } #[test] @@ -1023,11 +937,11 @@ mod tests { "2".to_string(), ), ]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(tp.cdc_enabled); - assert_eq!(tp.cdc_min_chunk_size, 256 * 1024); - assert_eq!(tp.cdc_max_chunk_size, 1024 * 1024); - assert_eq!(tp.cdc_norm_level, 2); + let tp = TableProperties::new(&props); + assert!(tp.cdc_enabled().unwrap()); + assert_eq!(tp.cdc_min_chunk_size().unwrap(), 256 * 1024); + assert_eq!(tp.cdc_max_chunk_size().unwrap(), 1024 * 1024); + assert_eq!(tp.cdc_norm_level().unwrap(), 2); } #[test] @@ -1042,8 +956,8 @@ mod tests { "-2".to_string(), ), ]); - let tp = TableProperties::try_from(&props).unwrap(); - assert_eq!(tp.cdc_norm_level, -2); + let tp = TableProperties::new(&props); + assert_eq!(tp.cdc_norm_level().unwrap(), -2); } #[test] @@ -1058,7 +972,9 @@ mod tests { "not_a_number".to_string(), ), ]); - let err = TableProperties::try_from(&props).unwrap_err(); + let err = TableProperties::new(&props) + .cdc_min_chunk_size() + .unwrap_err(); assert!( err.to_string().contains( "Invalid value for write.parquet.content-defined-chunking.min-chunk-size" @@ -1078,7 +994,7 @@ mod tests { "not_a_number".to_string(), ), ]); - let err = TableProperties::try_from(&props).unwrap_err(); + let err = TableProperties::new(&props).cdc_norm_level().unwrap_err(); assert!( err.to_string() .contains("Invalid value for write.parquet.content-defined-chunking.norm-level") @@ -1088,32 +1004,33 @@ mod tests { #[test] fn test_cdc_no_properties() { let props = HashMap::from([("some.other.property".to_string(), "value".to_string())]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(!tp.cdc_enabled); + let tp = TableProperties::new(&props); + assert!(!tp.cdc_enabled().unwrap()); } #[test] fn test_parquet_sizing_defaults() { - let tp = TableProperties::try_from(&HashMap::new()).unwrap(); + let props = HashMap::new(); + let tp = TableProperties::new(&props); // Default codec is zstd at its default level. assert_eq!( - tp.parquet_compression_codec, + tp.parquet_compression_codec().unwrap(), CompressionCodec::zstd_default() ); assert_eq!( - tp.parquet_row_group_size_bytes, + tp.parquet_row_group_size_bytes().unwrap(), TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT ); assert_eq!( - tp.parquet_page_size_bytes, + tp.parquet_page_size_bytes().unwrap(), TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT ); assert_eq!( - tp.parquet_page_row_limit, + tp.parquet_page_row_limit().unwrap(), TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT ); assert_eq!( - tp.parquet_dict_size_bytes, + tp.parquet_dict_size_bytes().unwrap(), TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT ); } @@ -1146,13 +1063,16 @@ mod tests { "131072".to_string(), ), ]); - let tp = TableProperties::try_from(&props).unwrap(); + let tp = TableProperties::new(&props); // Codec name and level are folded into a single CompressionCodec. - assert_eq!(tp.parquet_compression_codec, CompressionCodec::Gzip(4)); - assert_eq!(tp.parquet_row_group_size_bytes, 1048576); - assert_eq!(tp.parquet_page_size_bytes, 65536); - assert_eq!(tp.parquet_page_row_limit, 5000); - assert_eq!(tp.parquet_dict_size_bytes, 131072); + assert_eq!( + tp.parquet_compression_codec().unwrap(), + CompressionCodec::Gzip(4) + ); + assert_eq!(tp.parquet_row_group_size_bytes().unwrap(), 1048576); + assert_eq!(tp.parquet_page_size_bytes().unwrap(), 65536); + assert_eq!(tp.parquet_page_row_limit().unwrap(), 5000); + assert_eq!(tp.parquet_dict_size_bytes().unwrap(), 131072); } #[test] @@ -1161,7 +1081,9 @@ mod tests { TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES.to_string(), "not_a_number".to_string(), )]); - let err = TableProperties::try_from(&props).unwrap_err(); + let err = TableProperties::new(&props) + .parquet_row_group_size_bytes() + .unwrap_err(); assert_eq!(err.kind(), ErrorKind::DataInvalid); assert!( err.to_string() @@ -1187,8 +1109,12 @@ mod tests { TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(), name.to_string(), )]); - let tp = TableProperties::try_from(&props).unwrap(); - assert_eq!(tp.parquet_compression_codec, expected, "codec {name}"); + let tp = TableProperties::new(&props); + assert_eq!( + tp.parquet_compression_codec().unwrap(), + expected, + "codec {name}" + ); } } @@ -1206,8 +1132,11 @@ mod tests { "5".to_string(), ), ]); - let tp = TableProperties::try_from(&props).unwrap(); - assert_eq!(tp.parquet_compression_codec, CompressionCodec::Snappy); + let tp = TableProperties::new(&props); + assert_eq!( + tp.parquet_compression_codec().unwrap(), + CompressionCodec::Snappy + ); } #[test] @@ -1220,8 +1149,8 @@ mod tests { TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS.to_string(), f.to_string(), )]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(!tp.write_object_storage_partitioned_paths); + let tp = TableProperties::new(&props); + assert!(!tp.write_object_storage_partitioned_paths().unwrap()); } for t in true_variants { @@ -1229,8 +1158,8 @@ mod tests { TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS.to_string(), t.to_string(), )]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(tp.write_object_storage_partitioned_paths); + let tp = TableProperties::new(&props); + assert!(tp.write_object_storage_partitioned_paths().unwrap()); } } } diff --git a/crates/iceberg/src/transaction/expire_snapshots.rs b/crates/iceberg/src/transaction/expire_snapshots.rs index 2542dde68d..0a390d9829 100644 --- a/crates/iceberg/src/transaction/expire_snapshots.rs +++ b/crates/iceberg/src/transaction/expire_snapshots.rs @@ -97,7 +97,7 @@ impl ExpireSnapshotsAction { } /// Resolves the snapshots and refs to remove, following Java `RemoveSnapshots.internalApply`. - fn plan(&self, table: &Table, properties: &TableProperties) -> Result { + fn plan(&self, table: &Table, properties: &TableProperties<'_>) -> Result { // Matches Java `RemoveSnapshots.retainLast`, which requires at least one snapshot. if self.retain_last == Some(0) { return Err(Error::new( @@ -111,21 +111,24 @@ impl ExpireSnapshotsAction { // When a knob is not set explicitly, fall back to the table's `history.expire.*` properties, // matching Java `RemoveSnapshots`' constructor. With the default `max-snapshot-age-ms` (5 // days) the age path always runs, so even an explicit-id-only call applies the default cutoff. - let default_cutoff = self - .older_than_ms - .unwrap_or_else(|| now.saturating_sub(properties.max_snapshot_age_ms())); - let default_min_to_keep = self - .retain_last - .unwrap_or(properties.min_snapshots_to_keep()); + let default_cutoff = match self.older_than_ms { + Some(older_than_ms) => older_than_ms, + None => now.saturating_sub(properties.max_snapshot_age_ms()?), + }; + let default_min_to_keep = match self.retain_last { + Some(retain_last) => retain_last, + None => properties.min_snapshots_to_keep()?, + }; // Ref aging: `main` is always kept; any other ref whose head is older than its // `max_ref_age_ms` (defaulting to `history.expire.max-ref-age-ms`) is dropped, like Java's // `computeRetainedRefs`. + let default_max_ref_age_ms = properties.max_ref_age_ms()?; let mut removed_ref_names: Vec = vec![]; let mut retained_refs: Vec<&SnapshotReference> = vec![]; for (ref_name, snapshot_ref) in &metadata.refs { if ref_name == MAIN_BRANCH - || !Self::ref_aged_out(metadata, snapshot_ref, now, properties.max_ref_age_ms()) + || !Self::ref_aged_out(metadata, snapshot_ref, now, default_max_ref_age_ms) { retained_refs.push(snapshot_ref); } else { @@ -299,10 +302,10 @@ struct ExpirePlan { impl TransactionAction for ExpireSnapshotsAction { async fn commit(self: Arc, table: &Table) -> Result { let metadata = table.metadata(); - let properties = metadata.table_properties()?; + let properties = metadata.table_properties(); // Expiring metadata defeats a user's explicit decision to disable GC (Java refuses too). - if !properties.gc_enabled() { + if !properties.gc_enabled()? { return Err(Error::new( ErrorKind::DataInvalid, "Cannot expire snapshots: gc.enabled is false", diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 2345eb7845..3622471277 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -178,7 +178,7 @@ impl Transaction { return Ok(self.table); } - let table_props = self.table.metadata().table_properties()?; + let table_props = self.table.metadata().table_properties(); let backoff = Self::build_backoff(table_props)?; let tx = self; @@ -195,14 +195,14 @@ impl Transaction { .1 } - fn build_backoff(props: TableProperties) -> Result { + fn build_backoff(props: TableProperties<'_>) -> Result { Ok(ExponentialBuilder::new() - .with_min_delay(Duration::from_millis(props.commit_min_retry_wait_ms())) - .with_max_delay(Duration::from_millis(props.commit_max_retry_wait_ms())) + .with_min_delay(Duration::from_millis(props.commit_min_retry_wait_ms()?)) + .with_max_delay(Duration::from_millis(props.commit_max_retry_wait_ms()?)) .with_total_delay(Some(Duration::from_millis( - props.commit_total_retry_timeout_ms(), + props.commit_total_retry_timeout_ms()?, ))) - .with_max_times(props.commit_num_retries()) + .with_max_times(props.commit_num_retries()?) .with_factor(2.0) .build()) } diff --git a/crates/iceberg/src/writer/file_writer/location_generator.rs b/crates/iceberg/src/writer/file_writer/location_generator.rs index b8a612dc1a..26207d3e1d 100644 --- a/crates/iceberg/src/writer/file_writer/location_generator.rs +++ b/crates/iceberg/src/writer/file_writer/location_generator.rs @@ -21,7 +21,7 @@ use std::sync::Arc; use std::sync::atomic::AtomicU64; use crate::Result; -use crate::spec::{DataFileFormat, PartitionKey, TableMetadata, TableProperties}; +use crate::spec::{DataFileFormat, PartitionKey, TableMetadata}; use crate::util::location::strip_trailing_slash; /// `LocationGenerator` used to generate the location of data file. @@ -67,11 +67,12 @@ impl DefaultLocationGenerator { /// `{table_location}/data`. pub fn new(table_metadata: &TableMetadata) -> Result { let table_location = strip_trailing_slash(table_metadata.location()); - let prop = TableProperties::try_from(table_metadata.properties())?; + let prop = table_metadata.table_properties(); + let write_data_location = prop.write_data_location()?; + let write_folder_storage_location = prop.write_folder_storage_location()?; let data_location = strip_trailing_slash( - prop.write_data_location() - .clone() - .or_else(|| prop.write_folder_storage_location().clone()) + write_data_location + .or(write_folder_storage_location) .unwrap_or(format!("{table_location}{DEFAULT_DATA_DIR}")) .as_ref(), ) @@ -135,12 +136,14 @@ impl ObjectStorageLocationGenerator { /// `{table_location}/data`. pub fn new(table_metadata: &TableMetadata) -> Result { let table_location = strip_trailing_slash(table_metadata.location()); - let prop = TableProperties::try_from(table_metadata.properties())?; + let prop = table_metadata.table_properties(); + let write_data_location = prop.write_data_location()?; + let write_object_storage_location = prop.write_object_storage_location()?; + let write_folder_storage_location = prop.write_folder_storage_location()?; let storage_location = strip_trailing_slash( - prop.write_data_location() - .clone() - .or_else(|| prop.write_object_storage_location().clone()) - .or_else(|| prop.write_folder_storage_location().clone()) + write_data_location + .or(write_object_storage_location) + .or(write_folder_storage_location) .unwrap_or(format!("{table_location}{DEFAULT_DATA_DIR}")) .as_ref(), ) @@ -154,7 +157,7 @@ impl ObjectStorageLocationGenerator { Some(path_context(table_location)) }; - let include_partition_paths = prop.write_object_storage_partitioned_paths(); + let include_partition_paths = prop.write_object_storage_partitioned_paths()?; Ok(Self { storage_location, diff --git a/crates/iceberg/src/writer/file_writer/parquet_writer.rs b/crates/iceberg/src/writer/file_writer/parquet_writer.rs index 2d9ef2077c..968a7b7236 100644 --- a/crates/iceberg/src/writer/file_writer/parquet_writer.rs +++ b/crates/iceberg/src/writer/file_writer/parquet_writer.rs @@ -87,20 +87,27 @@ impl ParquetWriterBuilder { /// Build a `ParquetWriterBuilder` from Iceberg table properties and a /// schema, translating `write.parquet.*` settings into `WriterProperties` /// instead of using parquet-rs defaults. - pub fn from_table_properties(table_props: &TableProperties, schema: SchemaRef) -> Result { - let cdc = table_props.cdc_enabled().then_some(CdcOptions { - min_chunk_size: table_props.cdc_min_chunk_size(), - max_chunk_size: table_props.cdc_max_chunk_size(), - norm_level: table_props.cdc_norm_level(), - }); - let compression = parquet_compression(*table_props.parquet_compression_codec())?; + pub fn from_table_properties( + table_props: &TableProperties<'_>, + schema: SchemaRef, + ) -> Result { + let cdc = if table_props.cdc_enabled()? { + Some(CdcOptions { + min_chunk_size: table_props.cdc_min_chunk_size()?, + max_chunk_size: table_props.cdc_max_chunk_size()?, + norm_level: table_props.cdc_norm_level()?, + }) + } else { + None + }; + let compression = parquet_compression(table_props.parquet_compression_codec()?)?; let props = WriterProperties::builder() .set_content_defined_chunking(cdc) .set_compression(compression) - .set_max_row_group_bytes(Some(table_props.parquet_row_group_size_bytes())) - .set_data_page_size_limit(table_props.parquet_page_size_bytes()) - .set_data_page_row_count_limit(table_props.parquet_page_row_limit()) - .set_dictionary_page_size_limit(table_props.parquet_dict_size_bytes()) + .set_max_row_group_bytes(Some(table_props.parquet_row_group_size_bytes()?)) + .set_data_page_size_limit(table_props.parquet_page_size_bytes()?) + .set_data_page_row_count_limit(table_props.parquet_page_row_limit()?) + .set_dictionary_page_size_limit(table_props.parquet_dict_size_bytes()?) .build(); Ok(Self::new_with_match_mode(props, schema, FieldMatchMode::Id)) } @@ -1046,10 +1053,11 @@ mod tests { let output_file = file_io.new_output( location_gen.generate_location(None, &file_name_gen.generate_file_name()), )?; - let table_properties = table_props(HashMap::from([( + let raw_properties = HashMap::from([( TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(), "test-key".to_string(), - )])); + )]); + let table_properties = TableProperties::new(&raw_properties); let mut parquet_writer = ParquetWriterBuilder::from_table_properties(&table_properties, iceberg_schema.clone())? .with_encryption_manager(make_encryption_manager("test-key")) @@ -2546,20 +2554,18 @@ mod tests { ) } - fn table_props(entries: HashMap) -> TableProperties { - TableProperties::try_from(&entries).unwrap() - } - #[test] fn test_from_table_properties_no_cdc_by_default() { - let tp = table_props(HashMap::new()); + let raw_properties = HashMap::new(); + let tp = TableProperties::new(&raw_properties); let builder = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema()).unwrap(); assert!(builder.props.content_defined_chunking().is_none()); } #[tokio::test] async fn test_from_table_properties_without_encryption_writes_plaintext() { - let tp = table_props(HashMap::new()); + let raw_properties = HashMap::new(); + let tp = TableProperties::new(&raw_properties); let tmp = TempDir::new().unwrap(); let output = FileIO::new_with_fs() .new_output(format!("{}/plain.parquet", tmp.path().to_str().unwrap())) @@ -2589,7 +2595,7 @@ mod tests { // written file) keeps this a direct propagation check: every future // `write.parquet.*` option just adds an assertion on its corresponding // `WriterProperties` getter here. - let tp = table_props(HashMap::from([ + let raw_properties = HashMap::from([ ( TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), "true".to_string(), @@ -2606,7 +2612,8 @@ mod tests { TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(), "2".to_string(), ), - ])); + ]); + let tp = TableProperties::new(&raw_properties); let tmp = TempDir::new().unwrap(); let output = FileIO::new_with_fs() @@ -2632,7 +2639,8 @@ mod tests { fn test_from_table_properties_sizing_defaults() { // With no properties set, the writer must use Iceberg's defaults (which // differ from parquet-rs's own defaults), not parquet-rs's. - let tp = table_props(HashMap::new()); + let raw_properties = HashMap::new(); + let tp = TableProperties::new(&raw_properties); let props = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema()) .unwrap() .props; @@ -2662,7 +2670,7 @@ mod tests { #[test] fn test_from_table_properties_sizing_and_compression_overrides() { - let tp = table_props(HashMap::from([ + let raw_properties = HashMap::from([ ( TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES.to_string(), "1048576".to_string(), @@ -2687,7 +2695,8 @@ mod tests { TableProperties::PROPERTY_PARQUET_COMPRESSION_LEVEL.to_string(), "9".to_string(), ), - ])); + ]); + let tp = TableProperties::new(&raw_properties); let props = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema()) .unwrap() .props; @@ -2708,7 +2717,11 @@ mod tests { TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(), "bogus".to_string(), )]); - let err = TableProperties::try_from(&entries).unwrap_err(); + let err = ParquetWriterBuilder::from_table_properties( + &TableProperties::new(&entries), + cdc_test_schema(), + ) + .unwrap_err(); assert_eq!(err.kind(), ErrorKind::DataInvalid); assert!(err.to_string().contains("bogus")); } diff --git a/crates/iceberg/src/writer/file_writer/rolling_writer.rs b/crates/iceberg/src/writer/file_writer/rolling_writer.rs index f569436cb0..9a45cd0fb2 100644 --- a/crates/iceberg/src/writer/file_writer/rolling_writer.rs +++ b/crates/iceberg/src/writer/file_writer/rolling_writer.rs @@ -456,10 +456,11 @@ mod tests { let schema = make_test_schema()?; - let table_properties = TableProperties::try_from(&HashMap::from([( + let raw_properties = HashMap::from([( TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(), "test-key".to_string(), - )]))?; + )]); + let table_properties = TableProperties::new(&raw_properties); let parquet_writer_builder = ParquetWriterBuilder::from_table_properties(&table_properties, Arc::new(schema))? .with_encryption_manager(make_encryption_manager("test-key")); diff --git a/crates/integrations/datafusion/src/physical_plan/write.rs b/crates/integrations/datafusion/src/physical_plan/write.rs index 8e680b3d92..3d2ea32226 100644 --- a/crates/integrations/datafusion/src/physical_plan/write.rs +++ b/crates/integrations/datafusion/src/physical_plan/write.rs @@ -202,15 +202,14 @@ impl ExecutionPlan for IcebergWriteExec { let format_version = self.table.metadata().format_version(); // Get typed table properties - let table_props = self - .table - .metadata() - .table_properties() - .map_err(to_datafusion_error)?; + let table_props = self.table.metadata().table_properties(); // Check data file format - let file_format = DataFileFormat::from_str(table_props.write_format_default()) + let write_format_default = table_props + .write_format_default() .map_err(to_datafusion_error)?; + let file_format = + DataFileFormat::from_str(&write_format_default).map_err(to_datafusion_error)?; if file_format != DataFileFormat::Parquet { return Err(to_datafusion_error(Error::new( ErrorKind::FeatureUnsupported, @@ -231,7 +230,9 @@ impl ExecutionPlan for IcebergWriteExec { parquet_file_writer_builder = parquet_file_writer_builder.with_encryption_manager(encryption_manager.clone()); } - let target_file_size = table_props.write_target_file_size_bytes(); + let target_file_size = table_props + .write_target_file_size_bytes() + .map_err(to_datafusion_error)?; let file_io = self.table.file_io().clone(); // todo location_gen and file_name_gen should be configurable @@ -250,7 +251,9 @@ impl ExecutionPlan for IcebergWriteExec { let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); // Create TaskWriter - let fanout_enabled = table_props.write_datafusion_fanout_enabled(); + let fanout_enabled = table_props + .write_datafusion_fanout_enabled() + .map_err(to_datafusion_error)?; let schema = self.table.metadata().current_schema().clone(); let partition_spec = self.table.metadata().default_partition_spec().clone(); let task_writer = TaskWriter::try_new(