variant: Integrate datafusion-variant into Datafusion

Cargo.toml

@@ -37,6 +37,7 @@ members = [
     "datafusion/functions-aggregate-common",
     "datafusion/functions-table",
     "datafusion/functions-nested",
+    "datafusion/functions-variant",
     "datafusion/functions-window",
     "datafusion/functions-window-common",
     "datafusion/optimizer",
@@ -142,6 +143,7 @@ datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", vers
 datafusion-functions-aggregate-common = { path = "datafusion/functions-aggregate-common", version = "54.0.0" }
 datafusion-functions-nested = { path = "datafusion/functions-nested", version = "54.0.0", default-features = false }
 datafusion-functions-table = { path = "datafusion/functions-table", version = "54.0.0" }
+datafusion-functions-variant = { path = "datafusion/functions-variant", version = "54.0.0", default-features = false }
 datafusion-functions-window = { path = "datafusion/functions-window", version = "54.0.0" }
 datafusion-functions-window-common = { path = "datafusion/functions-window-common", version = "54.0.0" }
 datafusion-macros = { path = "datafusion/macros", version = "54.0.0" }
@@ -183,6 +185,9 @@ parquet = { version = "59.0.0", default-features = false, features = [
     "async",
     "object_store",
 ] }
+parquet-variant = { version = "59.0.0", default-features = false }
+parquet-variant-compute = { version = "59.0.0", default-features = false }
+parquet-variant-json = { version = "59.0.0", default-features = false }
 pbjson = { version = "0.9.0" }
 pbjson-types = "0.9"
 pin-project = "1"

datafusion/core/src/execution/session_state_defaults.rs

@@ -159,7 +159,7 @@ impl SessionStateDefaults {
         file_formats
     }
 
-    /// registers all builtin functions - scalar, array and aggregate
+    /// registers all builtin functions - scalar, array, and aggregate
     pub fn register_builtin_functions(state: &mut SessionState) {
         Self::register_scalar_functions(state);
         Self::register_array_functions(state);

datafusion/functions-variant/Cargo.toml

@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "datafusion-functions-variant"
+description = "Variant type Function packages for the DataFusion query engine"
+keywords = ["datafusion", "logical", "plan", "expressions", "variant"]
+readme = "README.md"
+version = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+repository = { workspace = true }
+license = { workspace = true }
+authors = { workspace = true }
+rust-version = { workspace = true }
+
+[package.metadata.docs.rs]
+all-features = true
+
+# Note: add additional linter rules in lib.rs.
+# Rust does not support workspace + new linter rules in subcrates yet
+# https://github.com/rust-lang/cargo/issues/13157
+[lints]
+workspace = true
+
+[lib]
+name = "datafusion_functions_variant"
+
+[features]
+default = []
+core = ["datafusion"]
+
+[dependencies]
+arrow = { workspace = true }
+arrow-schema = { workspace = true }
+# optional dependency for SessionStateBuilderVariant extension trait
+datafusion = { workspace = true, optional = true, default-features = false }
+datafusion-common = { workspace = true }
+datafusion-execution = { workspace = true }
+datafusion-expr = { workspace = true, default-features = false }
+log = { workspace = true }
+parquet-variant = { workspace = true }
+parquet-variant-compute = { workspace = true }
+parquet-variant-json = { workspace = true }
+
+[dev-dependencies]
+# for SessionStateBuilderVariant tests
+datafusion = { workspace = true, default-features = false, features = ["sql"] }
+serde_json = { workspace = true }

datafusion/functions-variant/LICENSE.txt

@@ -0,0 +1 @@
+../../LICENSE.txt

datafusion/functions-variant/NOTICE.txt

@@ -0,0 +1 @@
+../../NOTICE.txt

datafusion/functions-variant/README.md

@@ -0,0 +1,39 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Apache DataFusion Variant Function Library
+
+[Apache DataFusion] is an extensible query execution framework, written in Rust, that uses [Apache Arrow] as its in-memory format.
+
+This crate contains functions for working with the [Parquet Variant] logical
+type, such as `parse_json`, `variant_get`, `variant_object_keys`, and friends.
+Variant values are stored as Arrow `StructArray`s carrying the `VariantType`
+extension type and are produced/consumed by the [`parquet-variant`] family of
+crates.
+
+This crate is not a dependency of the [`datafusion`] crate. To register the
+Variant functions on a DataFusion session, enable the `core` feature and use
+the `SessionStateBuilderVariant::with_variant_features` extension method on
+`SessionStateBuilder`.
+
+[apache arrow]: https://arrow.apache.org/
+[apache datafusion]: https://datafusion.apache.org/
+[parquet variant]: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md
+[`parquet-variant`]: https://crates.io/crates/parquet-variant
+[`datafusion`]: https://crates.io/crates/datafusion