typetag serde for StorageFactory

Author: CTTY

crates/iceberg/src/io/file_io.rs

@@ -607,10 +607,16 @@ mod tests {
         }
     }
 
+    fn default_received_props() -> Arc<Mutex<HashMap<String, String>>> {
+        Arc::new(Mutex::new(HashMap::new()))
+    }
+
     // Test storage factory
-    #[derive(Debug)]
+    #[derive(Debug, Serialize, Deserialize)]
     struct TestStorageFactory {
+        #[serde(skip, default = "default_written")]
         written: Arc<Mutex<Vec<String>>>,
+        #[serde(skip, default = "default_received_props")]
         received_props: Arc<Mutex<HashMap<String, String>>>,
     }
 
@@ -624,6 +630,7 @@ mod tests {
         }
     }
 
+    #[typetag::serde]
     impl StorageFactory for TestStorageFactory {
         fn build(
             &self,

crates/iceberg/src/io/storage.rs

@@ -93,14 +93,19 @@ pub trait Storage: Debug + Send + Sync {
 /// properties and extensions. Each storage backend (S3, GCS, etc.) provides its own
 /// factory implementation.
 ///
+/// The trait supports serialization via `typetag`, allowing factory instances to be
+/// serialized and deserialized across process boundaries.
+///
 /// # Example
 ///
 /// ```rust,ignore
 /// use iceberg::io::{StorageFactory, Extensions};
 /// use std::collections::HashMap;
 ///
+/// #[derive(Debug, Serialize, Deserialize)]
 /// struct MyStorageFactory;
 ///
+/// #[typetag::serde]
 /// impl StorageFactory for MyStorageFactory {
 ///     fn build(
 ///         &self,
@@ -112,6 +117,7 @@ pub trait Storage: Debug + Send + Sync {
 ///     }
 /// }
 /// ```
+#[typetag::serde(tag = "type")]
 pub trait StorageFactory: Debug + Send + Sync {
     /// Create a new storage instance with the given properties and extensions.
     ///

crates/iceberg/src/io/storage_azdls.rs

@@ -688,9 +688,10 @@ impl Storage for OpenDALAzdlsStorage {
 }
 
 /// Factory for Azure Data Lake Storage
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct OpenDALAzdlsStorageFactory;
 
+#[typetag::serde]
 impl StorageFactory for OpenDALAzdlsStorageFactory {
     fn build(
         &self,

crates/iceberg/src/io/storage_fs.rs

@@ -123,9 +123,10 @@ impl Storage for OpenDALFsStorage {
 }
 
 /// Factory for OpenDAL Filesystem storage
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct OpenDALFsStorageFactory;
 
+#[typetag::serde]
 impl StorageFactory for OpenDALFsStorageFactory {
     fn build(
         &self,
@@ -158,4 +159,21 @@ mod tests {
         // Verify the type is correct
         assert!(format!("{:?}", deserialized).contains("OpenDALFsStorage"));
     }
+
+    #[test]
+    fn test_fs_factory_serialization() {
+        use crate::io::StorageFactory;
+
+        // Create a factory instance
+        let factory: Box<dyn StorageFactory> = Box::new(OpenDALFsStorageFactory);
+
+        // Serialize the factory
+        let serialized = serde_json::to_string(&factory).unwrap();
+
+        // Deserialize the factory
+        let deserialized: Box<dyn StorageFactory> = serde_json::from_str(&serialized).unwrap();
+
+        // Verify the type is correct
+        assert!(format!("{:?}", deserialized).contains("OpenDALFsStorageFactory"));
+    }
 }

crates/iceberg/src/io/storage_gcs.rs

@@ -200,9 +200,10 @@ impl Storage for OpenDALGcsStorage {
 }
 
 /// Factory for OpenDAL GCS storage
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct OpenDALGcsStorageFactory;
 
+#[typetag::serde]
 impl StorageFactory for OpenDALGcsStorageFactory {
     fn build(
         &self,

crates/iceberg/src/io/storage_memory.rs

@@ -141,9 +141,10 @@ impl Storage for OpenDALMemoryStorage {
 }
 
 /// Factory for OpenDAL Memory storage
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct OpenDALMemoryStorageFactory;
 
+#[typetag::serde]
 impl StorageFactory for OpenDALMemoryStorageFactory {
     fn build(
         &self,
@@ -176,4 +177,52 @@ mod tests {
         // Verify the type is correct
         assert!(format!("{:?}", deserialized).contains("OpenDALMemoryStorage"));
     }
+
+    #[test]
+    fn test_memory_factory_serialization() {
+        use crate::io::StorageFactory;
+
+        // Create a factory instance
+        let factory: Box<dyn StorageFactory> = Box::new(OpenDALMemoryStorageFactory);
+
+        // Serialize the factory
+        let serialized = serde_json::to_string(&factory).unwrap();
+
+        // Deserialize the factory
+        let deserialized: Box<dyn StorageFactory> = serde_json::from_str(&serialized).unwrap();
+
+        // Verify the type is correct
+        assert!(format!("{:?}", deserialized).contains("OpenDALMemoryStorageFactory"));
+    }
+
+    #[test]
+    fn test_memory_factory_to_storage_serialization() {
+        use crate::io::{Extensions, StorageFactory};
+
+        // Create a factory and build storage
+        let factory = OpenDALMemoryStorageFactory;
+        let storage = factory
+            .build(HashMap::new(), Extensions::default())
+            .unwrap();
+
+        // Serialize the storage
+        let storage_json = serde_json::to_string(&storage).unwrap();
+
+        // Deserialize the storage
+        let deserialized_storage: Box<dyn Storage> = serde_json::from_str(&storage_json).unwrap();
+
+        // Verify storage type
+        assert!(format!("{:?}", deserialized_storage).contains("OpenDALMemoryStorage"));
+
+        // Serialize the factory
+        let factory_boxed: Box<dyn StorageFactory> = Box::new(factory);
+        let factory_json = serde_json::to_string(&factory_boxed).unwrap();
+
+        // Deserialize the factory
+        let deserialized_factory: Box<dyn StorageFactory> =
+            serde_json::from_str(&factory_json).unwrap();
+
+        // Verify factory type
+        assert!(format!("{:?}", deserialized_factory).contains("OpenDALMemoryStorageFactory"));
+    }
 }

crates/iceberg/src/io/storage_oss.rs

@@ -160,9 +160,10 @@ impl Storage for OpenDALOssStorage {
 }
 
 /// Factory for OpenDAL OSS storage
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct OpenDALOssStorageFactory;
 
+#[typetag::serde]
 impl StorageFactory for OpenDALOssStorageFactory {
     fn build(
         &self,

crates/iceberg/src/io/storage_s3.rs

@@ -328,9 +328,10 @@ impl Storage for OpenDALS3Storage {
 }
 
 /// Factory for S3 storage
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct OpenDALS3StorageFactory;
 
+#[typetag::serde]
 impl StorageFactory for OpenDALS3StorageFactory {
     fn build(
         &self,
@@ -383,4 +384,21 @@ mod tests {
         // Verify the type is correct
         assert!(format!("{:?}", deserialized).contains("OpenDALS3Storage"));
     }
+
+    #[test]
+    fn test_s3_factory_serialization() {
+        use crate::io::StorageFactory;
+
+        // Create a factory instance
+        let factory: Box<dyn StorageFactory> = Box::new(OpenDALS3StorageFactory);
+
+        // Serialize the factory
+        let serialized = serde_json::to_string(&factory).unwrap();
+
+        // Deserialize the factory
+        let deserialized: Box<dyn StorageFactory> = serde_json::from_str(&serialized).unwrap();
+
+        // Verify the type is correct
+        assert!(format!("{:?}", deserialized).contains("OpenDALS3StorageFactory"));
+    }
 }