diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4b279830..db3429d1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,7 @@ jobs: strategy: fail-fast: false matrix: - producer: [datafusion, duckdb, ibis, isthmus] + producer: [datafusion, duckdb, ibis, isthmus, spark] steps: - name: Checkout repository uses: actions/checkout@v4 diff --git a/.gitignore b/.gitignore index bd6ad262..59cc0bdf 100644 --- a/.gitignore +++ b/.gitignore @@ -130,3 +130,6 @@ dmypy.json # PyCharm .idea/ + +jars/ +substrait_consumer/data/ diff --git a/requirements-unlocked.txt b/requirements-unlocked.txt index dde9b5a4..34549fbb 100644 --- a/requirements-unlocked.txt +++ b/requirements-unlocked.txt @@ -7,6 +7,7 @@ JPype1 pandas protobuf==5.28.3 pyarrow +pyspark pytest pytest-csv pytest-snapshot diff --git a/requirements.txt b/requirements.txt index da1f4088..1588a348 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ parsy==2.1 pluggy==1.5.0 protobuf==5.28.3 pyarrow==18.1.0 +pyspark==3.5.5 pytest==8.3.3 pytest-csv==3.0.0 pytest-snapshot==0.9.0 diff --git a/substrait-java b/substrait-java index 49b02100..7ebb5c6a 160000 --- a/substrait-java +++ b/substrait-java @@ -1 +1 @@ -Subproject commit 49b02100cfb9f15ac01ad37d592f960d24e6a175 +Subproject commit 7ebb5c6aec8b7bff71273b6afd3f7247705c84bd diff --git a/substrait_consumer/conftest.py b/substrait_consumer/conftest.py index 026557e0..39cc43ae 100644 --- a/substrait_consumer/conftest.py +++ b/substrait_consumer/conftest.py @@ -12,6 +12,7 @@ from substrait_consumer.producers.duckdb_producer import DuckDBProducer from substrait_consumer.producers.ibis_producer import IbisProducer from substrait_consumer.producers.isthmus_producer import IsthmusProducer +from substrait_consumer.producers.spark_producer import SparkProducer @pytest.fixture @@ -105,7 +106,7 @@ def saveplan(request): PRODUCERS = { cls.name(): cls - for cls in [DataFusionProducer, DuckDBProducer, IbisProducer, IsthmusProducer] + for cls in [DataFusionProducer, DuckDBProducer, IbisProducer, IsthmusProducer, SparkProducer] } CONSUMERS = { cls.name(): cls for cls in [AceroConsumer, DataFusionConsumer, DuckDBConsumer] diff --git a/substrait_consumer/producers/spark_producer.py b/substrait_consumer/producers/spark_producer.py new file mode 100644 index 00000000..a06ce0d2 --- /dev/null +++ b/substrait_consumer/producers/spark_producer.py @@ -0,0 +1,82 @@ +from pyspark.sql import SparkSession, DataFrame +from pathlib import Path +import os +import substrait_validator as sv +from .producer import SQLProducer + +class SparkProducer(SQLProducer): + """ + Adapts the Spark Substrait producer to the test framework. + """ + + @classmethod + def name(self): + return "spark" + + def __init__(self): + jars = "io.substrait:core:0.66.0,io.substrait:spark:0.66.0,com.google.protobuf:protobuf-java-util:4.33.0" + self._spark = SparkSession.builder.master("local").appName("SparkProducer").config("spark.jars.packages", jars).getOrCreate() + + def _setup( + self, db_connection, local_files: dict[str, str], named_tables: dict[str, str] + ): + self._tables = named_tables.keys() + + # since Spark always stores the absolute path name in its filesystem read relation, + # we need to add a symlink to a fixed temp directory to avoid different path names on + # different systems. + temp_dir = "/tmp/substrait-io" + link = temp_dir + "/consumer-testing" + Path(temp_dir).mkdir(parents=True, exist_ok=True) + if os.path.exists(link): + os.remove(link) + cwd = os.getcwd() + os.symlink(cwd, link, target_is_directory=True) + + for name, path in named_tables.items(): + table = self._spark.read.load(path.replace(cwd, link)) + table.createOrReplaceTempView(name) + + def _produce_substrait(self, sql_query: str, validate=False) -> str: + """ + Produce the Spark substrait plan using the given SQL query. + + Parameters: + sql_query: + SQL query. + Returns: + Substrait query plan in json format. + """ + + df = self._spark.sql(sql_query) + jvm = self._spark.sparkContext._jvm + + sparkPlan = df._jdf.queryExecution().optimizedPlan() + + toSubstrait = jvm.io.substrait.spark.logical.ToSubstraitRel() + substrait_plan = toSubstrait.convert(sparkPlan) + + proto_converter = jvm.io.substrait.plan.PlanProtoConverter() + proto_plan = proto_converter.toProto(substrait_plan) + proto_json = jvm.com.google.protobuf.util.JsonFormat.printer().print(proto_plan) + + if validate: + config = sv.Config() + # Warning: cannot automatically determine whether plan version + # is compatible with the Substrait version + config.override_diagnostic_level(7, "info", "info") # warning + # Warning: did not attempt to resolve YAML: configured recursion + # limit for URI resolution has been reached + config.override_diagnostic_level(2001, "info", "info") + sv.check_plan_valid(proto_json, config) + + return proto_json + + def _format_sql(self, sql_query: str) -> str: + # The table names are enclosed in single quotes (i.e. string literals) + # These need to be removed because Spark won't tolerate them. + sql = sql_query + for table in ("{"+ t + "}" for t in self._tables): + sql = sql.replace(f"'{table}'", table) + + return sql diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/acos-spark_outcome.txt b/substrait_consumer/snapshots/producer/function/arithmetic/acos-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/acos-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/acos-spark_plan.json b/substrait_consumer/snapshots/producer/function/arithmetic/acos-spark_plan.json new file mode 100644 index 00000000..dd37062b --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/acos-spark_plan.json @@ -0,0 +1,208 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 2, + "uri": "/functions_arithmetic.yaml" + }, { + "extensionUriAnchor": 1, + "uri": "/functions_rounding.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "round:fp64_i32", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "acos:fp64", + "extensionUrnReference": 2 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16] + }, + "hint": { + "outputNames": ["ACOS_TAX"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "scalarFunction": { + "functionReference": 1, + "outputType": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "cast": { + "type": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "input": { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }, + "failureBehavior": "FAILURE_BEHAVIOR_THROW_EXCEPTION" + } + } + }] + } + } + }, { + "value": { + "literal": { + "i32": 2 + } + } + }] + } + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["ACOS_TAX"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_rounding" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_arithmetic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/add-spark_outcome.txt b/substrait_consumer/snapshots/producer/function/arithmetic/add-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/add-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/add-spark_plan.json b/substrait_consumer/snapshots/producer/function/arithmetic/add-spark_plan.json new file mode 100644 index 00000000..841a831e --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/add-spark_plan.json @@ -0,0 +1,147 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_arithmetic.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "add:i64_i64", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6, 7] + }, + "hint": { + "outputNames": ["PS_PARTKEY", "PS_SUPPKEY", "ADD_KEY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["PS_PARTKEY", "PS_SUPPKEY", "ADD_KEY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_arithmetic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/asin-spark_outcome.txt b/substrait_consumer/snapshots/producer/function/arithmetic/asin-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/asin-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/asin-spark_plan.json b/substrait_consumer/snapshots/producer/function/arithmetic/asin-spark_plan.json new file mode 100644 index 00000000..6cc8b42e --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/asin-spark_plan.json @@ -0,0 +1,208 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 2, + "uri": "/functions_arithmetic.yaml" + }, { + "extensionUriAnchor": 1, + "uri": "/functions_rounding.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "round:fp64_i32", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "asin:fp64", + "extensionUrnReference": 2 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16] + }, + "hint": { + "outputNames": ["ASIN_TAX"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "scalarFunction": { + "functionReference": 1, + "outputType": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "cast": { + "type": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "input": { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }, + "failureBehavior": "FAILURE_BEHAVIOR_THROW_EXCEPTION" + } + } + }] + } + } + }, { + "value": { + "literal": { + "i32": 2 + } + } + }] + } + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["ASIN_TAX"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_rounding" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_arithmetic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/atan-spark_outcome.txt b/substrait_consumer/snapshots/producer/function/arithmetic/atan-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/atan-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/atan-spark_plan.json b/substrait_consumer/snapshots/producer/function/arithmetic/atan-spark_plan.json new file mode 100644 index 00000000..f8bdc9f1 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/atan-spark_plan.json @@ -0,0 +1,208 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 2, + "uri": "/functions_arithmetic.yaml" + }, { + "extensionUriAnchor": 1, + "uri": "/functions_rounding.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "round:fp64_i32", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "atan:fp64", + "extensionUrnReference": 2 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16] + }, + "hint": { + "outputNames": ["ATAN_TAX"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "scalarFunction": { + "functionReference": 1, + "outputType": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "cast": { + "type": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "input": { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }, + "failureBehavior": "FAILURE_BEHAVIOR_THROW_EXCEPTION" + } + } + }] + } + } + }, { + "value": { + "literal": { + "i32": 2 + } + } + }] + } + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["ATAN_TAX"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_rounding" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_arithmetic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/atan2-spark_outcome.txt b/substrait_consumer/snapshots/producer/function/arithmetic/atan2-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/atan2-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/atan2-spark_plan.json b/substrait_consumer/snapshots/producer/function/arithmetic/atan2-spark_plan.json new file mode 100644 index 00000000..9329a43d --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/atan2-spark_plan.json @@ -0,0 +1,230 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 2, + "uri": "/functions_arithmetic.yaml" + }, { + "extensionUriAnchor": 1, + "uri": "/functions_rounding.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "round:fp64_i32", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "atan2:fp64_fp64", + "extensionUrnReference": 2 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16] + }, + "hint": { + "outputNames": ["ATAN2_TAX"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "scalarFunction": { + "functionReference": 1, + "outputType": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "cast": { + "type": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "input": { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }, + "failureBehavior": "FAILURE_BEHAVIOR_THROW_EXCEPTION" + } + } + }, { + "value": { + "cast": { + "type": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "input": { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }, + "failureBehavior": "FAILURE_BEHAVIOR_THROW_EXCEPTION" + } + } + }] + } + } + }, { + "value": { + "literal": { + "i32": 2 + } + } + }] + } + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["ATAN2_TAX"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_rounding" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_arithmetic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/avg-spark_outcome.txt b/substrait_consumer/snapshots/producer/function/arithmetic/avg-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/avg-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/avg-spark_plan.json b/substrait_consumer/snapshots/producer/function/arithmetic/avg-spark_plan.json new file mode 100644 index 00000000..c9654add --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/avg-spark_plan.json @@ -0,0 +1,186 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 2, + "uri": "/functions_rounding_decimal.yaml" + }, { + "extensionUriAnchor": 1, + "uri": "/functions_arithmetic_decimal.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "avg:dec", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "round:dec_i32", + "extensionUrnReference": 2 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [1] + }, + "hint": { + "outputNames": ["AVG_SUPPLYCOST"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5] + }, + "hint": { + "outputNames": ["ps_supplycost"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + }], + "measures": [{ + "measure": { + "functionReference": 1, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 6, + "precision": 19, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "scalarFunction": { + "functionReference": 2, + "outputType": { + "decimal": { + "scale": 2, + "precision": 16, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "i32": 2 + } + } + }] + } + }] + } + }, + "names": ["AVG_SUPPLYCOST"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_rounding_decimal" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/cos-spark_outcome.txt b/substrait_consumer/snapshots/producer/function/arithmetic/cos-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/cos-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/cos-spark_plan.json b/substrait_consumer/snapshots/producer/function/arithmetic/cos-spark_plan.json new file mode 100644 index 00000000..edb10f98 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/cos-spark_plan.json @@ -0,0 +1,158 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 2, + "uri": "/functions_arithmetic.yaml" + }, { + "extensionUriAnchor": 1, + "uri": "/functions_rounding.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "round:fp64_i32", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "cos:fp64", + "extensionUrnReference": 2 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5] + }, + "hint": { + "outputNames": ["COS_SUPPLY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "scalarFunction": { + "functionReference": 1, + "outputType": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "cast": { + "type": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "input": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, + "failureBehavior": "FAILURE_BEHAVIOR_THROW_EXCEPTION" + } + } + }] + } + } + }, { + "value": { + "literal": { + "i32": 2 + } + } + }] + } + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["COS_SUPPLY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_rounding" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_arithmetic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/count-spark_outcome.txt b/substrait_consumer/snapshots/producer/function/arithmetic/count-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/count-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/count-spark_plan.json b/substrait_consumer/snapshots/producer/function/arithmetic/count-spark_plan.json new file mode 100644 index 00000000..3dabe56f --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/count-spark_plan.json @@ -0,0 +1,151 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_aggregate_generic.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "count:any", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [1] + }, + "hint": { + "outputNames": ["COUNT_SUPPLYCOST"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5] + }, + "hint": { + "outputNames": ["ps_supplycost"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + }], + "measures": [{ + "measure": { + "functionReference": 1, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "i64": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["COUNT_SUPPLYCOST"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_aggregate_generic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/count_star-spark_outcome.txt b/substrait_consumer/snapshots/producer/function/arithmetic/count_star-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/count_star-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/count_star-spark_plan.json b/substrait_consumer/snapshots/producer/function/arithmetic/count_star-spark_plan.json new file mode 100644 index 00000000..38988a86 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/count_star-spark_plan.json @@ -0,0 +1,133 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_aggregate_generic.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "count:any", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [1] + }, + "hint": { + "outputNames": ["count(1)"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + }, + "hint": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + } + } + }, + "groupings": [{ + }], + "measures": [{ + "measure": { + "functionReference": 1, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "i64": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "literal": { + "i32": 1 + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["count(1)"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_aggregate_generic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/divide-spark_outcome.txt b/substrait_consumer/snapshots/producer/function/arithmetic/divide-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/divide-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/divide-spark_plan.json b/substrait_consumer/snapshots/producer/function/arithmetic/divide-spark_plan.json new file mode 100644 index 00000000..090e46b8 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/divide-spark_plan.json @@ -0,0 +1,141 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_arithmetic.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "divide:fp64_fp64", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6] + }, + "hint": { + "outputNames": ["PS_PARTKEY", "DIVIDE_KEY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "cast": { + "type": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "input": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "failureBehavior": "FAILURE_BEHAVIOR_THROW_EXCEPTION" + } + } + }, { + "value": { + "literal": { + "fp64": 10.0 + } + } + }] + } + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["PS_PARTKEY", "DIVIDE_KEY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_arithmetic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/exp-spark_outcome.txt b/substrait_consumer/snapshots/producer/function/arithmetic/exp-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/exp-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/exp-spark_plan.json b/substrait_consumer/snapshots/producer/function/arithmetic/exp-spark_plan.json new file mode 100644 index 00000000..6216edd0 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/exp-spark_plan.json @@ -0,0 +1,166 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 2, + "uri": "/functions_arithmetic.yaml" + }, { + "extensionUriAnchor": 1, + "uri": "/functions_rounding.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "round:fp64_i32", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "exp:fp64", + "extensionUrnReference": 2 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6] + }, + "hint": { + "outputNames": ["PS_PARTKEY", "EXP_KEY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "cast": { + "type": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "input": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "failureBehavior": "FAILURE_BEHAVIOR_THROW_EXCEPTION" + } + } + }] + } + } + }, { + "value": { + "literal": { + "i32": 2 + } + } + }] + } + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["PS_PARTKEY", "EXP_KEY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_rounding" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_arithmetic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/max-spark_outcome.txt b/substrait_consumer/snapshots/producer/function/arithmetic/max-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/max-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/max-spark_plan.json b/substrait_consumer/snapshots/producer/function/arithmetic/max-spark_plan.json new file mode 100644 index 00000000..c6393be3 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/max-spark_plan.json @@ -0,0 +1,153 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_arithmetic_decimal.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "max:dec", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [1] + }, + "hint": { + "outputNames": ["MAX_SUPPLYCOST"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5] + }, + "hint": { + "outputNames": ["ps_supplycost"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + }], + "measures": [{ + "measure": { + "functionReference": 1, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["MAX_SUPPLYCOST"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/min-spark_outcome.txt b/substrait_consumer/snapshots/producer/function/arithmetic/min-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/min-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/min-spark_plan.json b/substrait_consumer/snapshots/producer/function/arithmetic/min-spark_plan.json new file mode 100644 index 00000000..ca539f62 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/min-spark_plan.json @@ -0,0 +1,153 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_arithmetic_decimal.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "min:dec", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [1] + }, + "hint": { + "outputNames": ["MIN_SUPPLYCOST"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5] + }, + "hint": { + "outputNames": ["ps_supplycost"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + }], + "measures": [{ + "measure": { + "functionReference": 1, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["MIN_SUPPLYCOST"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/modulus-spark_outcome.txt b/substrait_consumer/snapshots/producer/function/arithmetic/modulus-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/modulus-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/modulus-spark_plan.json b/substrait_consumer/snapshots/producer/function/arithmetic/modulus-spark_plan.json new file mode 100644 index 00000000..a2bd2237 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/modulus-spark_plan.json @@ -0,0 +1,131 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_arithmetic.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "modulus:i64_i64", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6] + }, + "hint": { + "outputNames": ["PS_PARTKEY", "MODULUS_KEY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "i64": "10" + } + } + }] + } + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["PS_PARTKEY", "MODULUS_KEY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_arithmetic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/multiply-spark_outcome.txt b/substrait_consumer/snapshots/producer/function/arithmetic/multiply-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/multiply-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/multiply-spark_plan.json b/substrait_consumer/snapshots/producer/function/arithmetic/multiply-spark_plan.json new file mode 100644 index 00000000..e50be5cd --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/multiply-spark_plan.json @@ -0,0 +1,131 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_arithmetic.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "multiply:i64_i64", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6] + }, + "hint": { + "outputNames": ["PS_PARTKEY", "MULTIPLY_KEY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "i64": "10" + } + } + }] + } + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["PS_PARTKEY", "MULTIPLY_KEY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_arithmetic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/power-spark_outcome.txt b/substrait_consumer/snapshots/producer/function/arithmetic/power-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/power-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/power-spark_plan.json b/substrait_consumer/snapshots/producer/function/arithmetic/power-spark_plan.json new file mode 100644 index 00000000..00ffd038 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/power-spark_plan.json @@ -0,0 +1,141 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_arithmetic.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "power:fp64_fp64", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6] + }, + "hint": { + "outputNames": ["PS_PARTKEY", "POWER_KEY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "cast": { + "type": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "input": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "failureBehavior": "FAILURE_BEHAVIOR_THROW_EXCEPTION" + } + } + }, { + "value": { + "literal": { + "fp64": 2.0 + } + } + }] + } + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["PS_PARTKEY", "POWER_KEY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_arithmetic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/sin-spark_outcome.txt b/substrait_consumer/snapshots/producer/function/arithmetic/sin-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/sin-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/sin-spark_plan.json b/substrait_consumer/snapshots/producer/function/arithmetic/sin-spark_plan.json new file mode 100644 index 00000000..924b8a7c --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/sin-spark_plan.json @@ -0,0 +1,158 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 2, + "uri": "/functions_arithmetic.yaml" + }, { + "extensionUriAnchor": 1, + "uri": "/functions_rounding.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "round:fp64_i32", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "sin:fp64", + "extensionUrnReference": 2 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5] + }, + "hint": { + "outputNames": ["SIN_SUPPLY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "scalarFunction": { + "functionReference": 1, + "outputType": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "cast": { + "type": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "input": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, + "failureBehavior": "FAILURE_BEHAVIOR_THROW_EXCEPTION" + } + } + }] + } + } + }, { + "value": { + "literal": { + "i32": 2 + } + } + }] + } + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["SIN_SUPPLY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_rounding" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_arithmetic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/sqrt-spark_outcome.txt b/substrait_consumer/snapshots/producer/function/arithmetic/sqrt-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/sqrt-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/sqrt-spark_plan.json b/substrait_consumer/snapshots/producer/function/arithmetic/sqrt-spark_plan.json new file mode 100644 index 00000000..74e529d2 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/sqrt-spark_plan.json @@ -0,0 +1,166 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 2, + "uri": "/functions_arithmetic.yaml" + }, { + "extensionUriAnchor": 1, + "uri": "/functions_rounding.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "round:fp64_i32", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "sqrt:fp64", + "extensionUrnReference": 2 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6] + }, + "hint": { + "outputNames": ["PS_PARTKEY", "SQRT_KEY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "cast": { + "type": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "input": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "failureBehavior": "FAILURE_BEHAVIOR_THROW_EXCEPTION" + } + } + }] + } + } + }, { + "value": { + "literal": { + "i32": 2 + } + } + }] + } + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["PS_PARTKEY", "SQRT_KEY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_rounding" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_arithmetic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/std_dev-spark_outcome.txt b/substrait_consumer/snapshots/producer/function/arithmetic/std_dev-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/std_dev-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/std_dev-spark_plan.json b/substrait_consumer/snapshots/producer/function/arithmetic/std_dev-spark_plan.json new file mode 100644 index 00000000..4fee52ed --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/std_dev-spark_plan.json @@ -0,0 +1,192 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_arithmetic.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_rounding.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "std_dev:fp64", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "round:fp64_i32", + "extensionUrnReference": 2 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [1] + }, + "hint": { + "outputNames": ["STDDEV_SUPPLYCOST"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5] + }, + "hint": { + "outputNames": ["ps_supplycost"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + }], + "measures": [{ + "measure": { + "functionReference": 1, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "cast": { + "type": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "input": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "failureBehavior": "FAILURE_BEHAVIOR_THROW_EXCEPTION" + } + } + }] + } + }] + } + }, + "expressions": [{ + "scalarFunction": { + "functionReference": 2, + "outputType": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "i32": 2 + } + } + }] + } + }] + } + }, + "names": ["STDDEV_SUPPLYCOST"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_rounding" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_arithmetic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/subtract-spark_outcome.txt b/substrait_consumer/snapshots/producer/function/arithmetic/subtract-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/subtract-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/subtract-spark_plan.json b/substrait_consumer/snapshots/producer/function/arithmetic/subtract-spark_plan.json new file mode 100644 index 00000000..0dc3f903 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/subtract-spark_plan.json @@ -0,0 +1,147 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_arithmetic.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "subtract:i64_i64", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6, 7] + }, + "hint": { + "outputNames": ["PS_PARTKEY", "PS_SUPPKEY", "SUBTRACT_KEY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["PS_PARTKEY", "PS_SUPPKEY", "SUBTRACT_KEY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_arithmetic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/sum-spark_outcome.txt b/substrait_consumer/snapshots/producer/function/arithmetic/sum-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/sum-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/sum-spark_plan.json b/substrait_consumer/snapshots/producer/function/arithmetic/sum-spark_plan.json new file mode 100644 index 00000000..d2c348aa --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/sum-spark_plan.json @@ -0,0 +1,153 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_arithmetic_decimal.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "sum:dec", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [1] + }, + "hint": { + "outputNames": ["SUM_SUPPLYCOST"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5] + }, + "hint": { + "outputNames": ["ps_supplycost"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + }], + "measures": [{ + "measure": { + "functionReference": 1, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 2, + "precision": 25, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["SUM_SUPPLYCOST"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/tan-spark_outcome.txt b/substrait_consumer/snapshots/producer/function/arithmetic/tan-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/tan-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/function/arithmetic/tan-spark_plan.json b/substrait_consumer/snapshots/producer/function/arithmetic/tan-spark_plan.json new file mode 100644 index 00000000..fd9d7dca --- /dev/null +++ b/substrait_consumer/snapshots/producer/function/arithmetic/tan-spark_plan.json @@ -0,0 +1,158 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 2, + "uri": "/functions_arithmetic.yaml" + }, { + "extensionUriAnchor": 1, + "uri": "/functions_rounding.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "round:fp64_i32", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "tan:fp64", + "extensionUrnReference": 2 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5] + }, + "hint": { + "outputNames": ["TAN_SUPPLY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "scalarFunction": { + "functionReference": 1, + "outputType": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "cast": { + "type": { + "fp64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "input": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, + "failureBehavior": "FAILURE_BEHAVIOR_THROW_EXCEPTION" + } + } + }] + } + } + }, { + "value": { + "literal": { + "i32": 2 + } + } + }] + } + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["TAN_SUPPLY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_rounding" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_arithmetic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q01-spark_outcome.txt b/substrait_consumer/snapshots/producer/integration/tpch/q01-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q01-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q01-spark_plan.json b/substrait_consumer/snapshots/producer/integration/tpch/q01-spark_plan.json new file mode 100644 index 00000000..d543b7ab --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q01-spark_plan.json @@ -0,0 +1,871 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 5, + "uri": "/functions_aggregate_generic.yaml" + }, { + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 4, + "uri": "/functions_arithmetic_decimal.yaml" + }, { + "extensionUriAnchor": 3, + "uri": "/functions_datetime.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 3, + "name": "lte:date_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 4, + "name": "sum:dec", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 5, + "name": "multiply:dec_dec", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 6, + "name": "subtract:dec_dec", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 7, + "name": "add:dec_dec", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 8, + "name": "avg:dec", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 5, + "functionAnchor": 9, + "name": "count:any", + "extensionUrnReference": 5 + } + }], + "relations": [{ + "root": { + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [10, 11, 12, 13, 14, 15, 16, 17, 18, 19] + }, + "hint": { + "outputNames": ["l_returnflag", "l_linestatus", "sum_qty", "sum_base_price", "sum_disc_price", "sum_charge", "avg_qty", "avg_price", "avg_disc", "count_order"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17, 18, 19, 20, 21] + }, + "hint": { + "outputNames": ["l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 10441 + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 8 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 9 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 4, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 2, + "precision": 25, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }, { + "measure": { + "functionReference": 4, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 2, + "precision": 25, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + }, { + "measure": { + "functionReference": 4, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 4, + "precision": 38, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "decimal": { + "scale": 4, + "precision": 32, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "decimal": { + "scale": 2, + "precision": 16, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "literal": { + "decimal": { + "value": "AQAAAAAAAAAAAAAAAAAAAA==", + "precision": 1 + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }] + } + }, { + "measure": { + "functionReference": 4, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 6, + "precision": 38, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "decimal": { + "scale": 6, + "precision": 38, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "decimal": { + "scale": 4, + "precision": 32, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "decimal": { + "scale": 2, + "precision": 16, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "literal": { + "decimal": { + "value": "AQAAAAAAAAAAAAAAAAAAAA==", + "precision": 1 + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 7, + "outputType": { + "decimal": { + "scale": 2, + "precision": 16, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "literal": { + "decimal": { + "value": "AQAAAAAAAAAAAAAAAAAAAA==", + "precision": 1 + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }] + } + }, { + "measure": { + "functionReference": 8, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 6, + "precision": 19, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }, { + "measure": { + "functionReference": 8, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 6, + "precision": 19, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + }, { + "measure": { + "functionReference": 8, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 6, + "precision": 19, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + }, { + "measure": { + "functionReference": 9, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "i64": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "literal": { + "i32": 1 + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 8 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 9 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "names": ["l_returnflag", "l_linestatus", "sum_qty", "sum_base_price", "sum_disc_price", "sum_charge", "avg_qty", "avg_price", "avg_disc", "count_order"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 4, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 3, + "urn": "extension:io.substrait:functions_datetime" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }, { + "extensionUrnAnchor": 5, + "urn": "extension:io.substrait:functions_aggregate_generic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q02-spark_outcome.txt b/substrait_consumer/snapshots/producer/integration/tpch/q02-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q02-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q02-spark_plan.json b/substrait_consumer/snapshots/producer/integration/tpch/q02-spark_plan.json new file mode 100644 index 00000000..fc61f61a --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q02-spark_plan.json @@ -0,0 +1,2651 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 3, + "uri": "/functions_string.yaml" + }, { + "extensionUriAnchor": 4, + "uri": "/functions_arithmetic_decimal.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 3, + "name": "equal:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 4, + "name": "ends_with:str_str", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 5, + "name": "min:dec", + "extensionUrnReference": 4 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [10, 11, 12, 13, 14, 15, 16, 17] + }, + "hint": { + "outputNames": ["s_acctbal", "s_name", "n_name", "p_partkey", "p_mfgr", "s_address", "s_phone", "s_comment"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [11, 12, 13, 14, 15, 16, 17, 18, 19] + }, + "hint": { + "outputNames": ["p_partkey", "p_mfgr", "s_name", "s_address", "s_phone", "s_acctbal", "s_comment", "n_name", "n_regionkey"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [10, 11, 12, 13, 14, 15, 16, 17] + }, + "hint": { + "outputNames": ["p_partkey", "p_mfgr", "s_name", "s_address", "s_nationkey", "s_phone", "s_acctbal", "s_comment"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [6, 7, 8] + }, + "hint": { + "outputNames": ["p_partkey", "p_mfgr", "ps_suppkey"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6, 7, 8] + }, + "hint": { + "outputNames": ["p_partkey", "p_mfgr", "ps_suppkey", "ps_supplycost"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [9, 10] + }, + "hint": { + "outputNames": ["p_partkey", "p_mfgr"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["p_partkey", "p_name", "p_mfgr", "p_brand", "p_type", "p_size", "p_container", "p_retailprice", "p_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/part.parquet", + "length": "695574", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "i32": 15 + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "BRASS" + } + } + }] + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6, 7] + }, + "hint": { + "outputNames": ["ps_partkey", "ps_suppkey", "ps_supplycost"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [2, 3] + }, + "hint": { + "outputNames": ["min(ps_supplycost)", "ps_partkey"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5] + }, + "hint": { + "outputNames": ["ps_partkey", "ps_supplycost"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6, 7] + }, + "hint": { + "outputNames": ["ps_partkey", "ps_supplycost", "n_regionkey"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6, 7] + }, + "hint": { + "outputNames": ["ps_partkey", "ps_supplycost", "s_nationkey"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6, 7] + }, + "hint": { + "outputNames": ["ps_partkey", "ps_suppkey", "ps_supplycost"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [7, 8] + }, + "hint": { + "outputNames": ["s_suppkey", "s_nationkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["s_suppkey", "s_name", "s_address", "s_nationkey", "s_phone", "s_acctbal", "s_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/supplier.parquet", + "length": "83257", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5] + }, + "hint": { + "outputNames": ["n_nationkey", "n_regionkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["n_nationkey", "n_name", "n_regionkey", "n_comment"], + "struct": { + "types": [{ + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/nation.parquet", + "length": "2248", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [3] + }, + "hint": { + "outputNames": ["r_regionkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["r_regionkey", "r_name", "r_comment"], + "struct": { + "types": [{ + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/region.parquet", + "length": "1058", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "EUROPE" + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 5, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "condition": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + } + }, + "expression": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["s_suppkey", "s_name", "s_address", "s_nationkey", "s_phone", "s_acctbal", "s_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/supplier.parquet", + "length": "83257", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expression": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 8 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 9 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5, 6] + }, + "hint": { + "outputNames": ["n_nationkey", "n_name", "n_regionkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["n_nationkey", "n_name", "n_regionkey", "n_comment"], + "struct": { + "types": [{ + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/nation.parquet", + "length": "2248", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 8 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 9 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [3] + }, + "hint": { + "outputNames": ["r_regionkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["r_regionkey", "r_name", "r_comment"], + "struct": { + "types": [{ + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/region.parquet", + "length": "1058", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "EUROPE" + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 8 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 9 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_DESC_NULLS_LAST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "offset": "0", + "count": "100" + } + }, + "names": ["s_acctbal", "s_name", "n_name", "p_partkey", "p_mfgr", "s_address", "s_phone", "s_comment"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 4, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }, { + "extensionUrnAnchor": 3, + "urn": "extension:io.substrait:functions_string" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q03-spark_outcome.txt b/substrait_consumer/snapshots/producer/integration/tpch/q03-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q03-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q03-spark_plan.json b/substrait_consumer/snapshots/producer/integration/tpch/q03-spark_plan.json new file mode 100644 index 00000000..fa45bdc0 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q03-spark_plan.json @@ -0,0 +1,1188 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 4, + "uri": "/functions_arithmetic_decimal.yaml" + }, { + "extensionUriAnchor": 3, + "uri": "/functions_datetime.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 3, + "name": "equal:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 4, + "name": "lt:date_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 5, + "name": "gt:date_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 6, + "name": "multiply:dec_dec", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 7, + "name": "subtract:dec_dec", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 8, + "name": "sum:dec", + "extensionUrnReference": 4 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5, 6, 7] + }, + "hint": { + "outputNames": ["l_orderkey", "revenue", "o_orderdate", "o_shippriority"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [6, 7, 8, 9, 10] + }, + "hint": { + "outputNames": ["o_orderdate", "o_shippriority", "l_orderkey", "l_extendedprice", "l_discount"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6, 7] + }, + "hint": { + "outputNames": ["o_orderkey", "o_orderdate", "o_shippriority"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [8] + }, + "hint": { + "outputNames": ["c_custkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer.parquet", + "length": "1263269", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "BUILDING" + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [9, 10, 11, 12] + }, + "hint": { + "outputNames": ["o_orderkey", "o_custkey", "o_orderdate", "o_shippriority"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders.parquet", + "length": "6003201", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 9204 + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17, 18] + }, + "hint": { + "outputNames": ["l_orderkey", "l_extendedprice", "l_discount"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 9204 + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 8, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 4, + "precision": 38, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "decimal": { + "scale": 4, + "precision": 32, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 7, + "outputType": { + "decimal": { + "scale": 2, + "precision": 16, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "literal": { + "decimal": { + "value": "AQAAAAAAAAAAAAAAAAAAAA==", + "precision": 1 + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_DESC_NULLS_LAST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["l_orderkey", "revenue", "o_orderdate", "o_shippriority"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 4, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 3, + "urn": "extension:io.substrait:functions_datetime" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q04-spark_outcome.txt b/substrait_consumer/snapshots/producer/integration/tpch/q04-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q04-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q04-spark_plan.json b/substrait_consumer/snapshots/producer/integration/tpch/q04-spark_plan.json new file mode 100644 index 00000000..ea5a5930 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q04-spark_plan.json @@ -0,0 +1,678 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 4, + "uri": "/functions_aggregate_generic.yaml" + }, { + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 3, + "uri": "/functions_datetime.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 3, + "name": "gte:date_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 4, + "name": "lt:date_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 5, + "name": "equal:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 6, + "name": "count:any", + "extensionUrnReference": 4 + } + }], + "relations": [{ + "root": { + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [2, 3] + }, + "hint": { + "outputNames": ["o_orderpriority", "order_count"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [2] + }, + "hint": { + "outputNames": ["o_orderpriority"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [9, 10] + }, + "hint": { + "outputNames": ["o_orderkey", "o_orderpriority"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders.parquet", + "length": "6003201", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 8582 + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 8674 + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [16] + }, + "hint": { + "outputNames": ["l_orderkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 11 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 12 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 11 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 12 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_LEFT_SEMI" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 6, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "i64": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "literal": { + "i32": 1 + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "names": ["o_orderpriority", "order_count"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 3, + "urn": "extension:io.substrait:functions_datetime" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }, { + "extensionUrnAnchor": 4, + "urn": "extension:io.substrait:functions_aggregate_generic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q05-spark_outcome.txt b/substrait_consumer/snapshots/producer/integration/tpch/q05-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q05-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q05-spark_plan.json b/substrait_consumer/snapshots/producer/integration/tpch/q05-spark_plan.json new file mode 100644 index 00000000..e9c52941 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q05-spark_plan.json @@ -0,0 +1,1842 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 4, + "uri": "/functions_arithmetic_decimal.yaml" + }, { + "extensionUriAnchor": 3, + "uri": "/functions_datetime.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 3, + "name": "gte:date_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 4, + "name": "lt:date_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 5, + "name": "equal:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 6, + "name": "multiply:dec_dec", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 7, + "name": "subtract:dec_dec", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 8, + "name": "sum:dec", + "extensionUrnReference": 4 + } + }], + "relations": [{ + "root": { + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [2, 3] + }, + "hint": { + "outputNames": ["n_name", "revenue"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6, 7] + }, + "hint": { + "outputNames": ["l_extendedprice", "l_discount", "n_name"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [6, 7, 8, 9] + }, + "hint": { + "outputNames": ["l_extendedprice", "l_discount", "n_name", "n_regionkey"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [6, 7, 8] + }, + "hint": { + "outputNames": ["l_extendedprice", "l_discount", "s_nationkey"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [6, 7, 8, 9] + }, + "hint": { + "outputNames": ["c_nationkey", "l_suppkey", "l_extendedprice", "l_discount"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5] + }, + "hint": { + "outputNames": ["c_nationkey", "o_orderkey"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [8, 9] + }, + "hint": { + "outputNames": ["c_custkey", "c_nationkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer.parquet", + "length": "1263269", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [9, 10] + }, + "hint": { + "outputNames": ["o_orderkey", "o_custkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders.parquet", + "length": "6003201", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 8766 + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 9131 + } + } + }] + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17, 18, 19] + }, + "hint": { + "outputNames": ["l_orderkey", "l_suppkey", "l_extendedprice", "l_discount"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [7, 8] + }, + "hint": { + "outputNames": ["s_suppkey", "s_nationkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["s_suppkey", "s_name", "s_address", "s_nationkey", "s_phone", "s_acctbal", "s_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/supplier.parquet", + "length": "83257", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5, 6] + }, + "hint": { + "outputNames": ["n_nationkey", "n_name", "n_regionkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["n_nationkey", "n_name", "n_regionkey", "n_comment"], + "struct": { + "types": [{ + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/nation.parquet", + "length": "2248", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [3] + }, + "hint": { + "outputNames": ["r_regionkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["r_regionkey", "r_name", "r_comment"], + "struct": { + "types": [{ + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/region.parquet", + "length": "1058", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "ASIA" + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 8, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 4, + "precision": 38, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "decimal": { + "scale": 4, + "precision": 32, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 7, + "outputType": { + "decimal": { + "scale": 2, + "precision": 16, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "literal": { + "decimal": { + "value": "AQAAAAAAAAAAAAAAAAAAAA==", + "precision": 1 + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_DESC_NULLS_LAST" + }] + } + }, + "names": ["n_name", "revenue"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 4, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 3, + "urn": "extension:io.substrait:functions_datetime" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q06-spark_outcome.txt b/substrait_consumer/snapshots/producer/integration/tpch/q06-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q06-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q06-spark_plan.json b/substrait_consumer/snapshots/producer/integration/tpch/q06-spark_plan.json new file mode 100644 index 00000000..e738a31a --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q06-spark_plan.json @@ -0,0 +1,638 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 4, + "uri": "/functions_arithmetic_decimal.yaml" + }, { + "extensionUriAnchor": 3, + "uri": "/functions_datetime.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 3, + "name": "gte:date_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 4, + "name": "lt:date_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 5, + "name": "gte:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 6, + "name": "lte:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 7, + "name": "lt:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 8, + "name": "multiply:dec_dec", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 9, + "name": "sum:dec", + "extensionUrnReference": 4 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [1] + }, + "hint": { + "outputNames": ["revenue"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17] + }, + "hint": { + "outputNames": ["l_extendedprice", "l_discount"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 8766 + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 9131 + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "decimal": { + "value": "BQAAAAAAAAAAAAAAAAAAAA==", + "precision": 15, + "scale": 2 + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "decimal": { + "value": "BwAAAAAAAAAAAAAAAAAAAA==", + "precision": 15, + "scale": 2 + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 7, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "decimal": { + "value": "YAkAAAAAAAAAAAAAAAAAAA==", + "precision": 15, + "scale": 2 + } + } + } + }] + } + } + }] + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + }], + "measures": [{ + "measure": { + "functionReference": 9, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 4, + "precision": 38, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 8, + "outputType": { + "decimal": { + "scale": 4, + "precision": 31, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["revenue"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 4, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 3, + "urn": "extension:io.substrait:functions_datetime" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q07-spark_outcome.txt b/substrait_consumer/snapshots/producer/integration/tpch/q07-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q07-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q07-spark_plan.json b/substrait_consumer/snapshots/producer/integration/tpch/q07-spark_plan.json new file mode 100644 index 00000000..d4104a3a --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q07-spark_plan.json @@ -0,0 +1,2203 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 4, + "uri": "/functions_arithmetic_decimal.yaml" + }, { + "extensionUriAnchor": 3, + "uri": "/functions_datetime.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 3, + "name": "gte:date_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 4, + "name": "lte:date_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 5, + "name": "equal:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 6, + "name": "or:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 7, + "name": "extract:req_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 8, + "name": "multiply:dec_dec", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 9, + "name": "subtract:dec_dec", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 10, + "name": "sum:dec", + "extensionUrnReference": 4 + } + }], + "relations": [{ + "root": { + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5, 6, 7] + }, + "hint": { + "outputNames": ["supp_nation", "cust_nation", "l_year", "revenue"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [7, 8, 9, 10] + }, + "hint": { + "outputNames": ["supp_nation", "cust_nation", "l_year", "volume"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [7, 8, 9, 10, 11] + }, + "hint": { + "outputNames": ["l_extendedprice", "l_discount", "l_shipdate", "c_nationkey", "n_name"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [7, 8, 9, 10, 11] + }, + "hint": { + "outputNames": ["s_nationkey", "l_extendedprice", "l_discount", "l_shipdate", "c_nationkey"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [7, 8, 9, 10, 11] + }, + "hint": { + "outputNames": ["s_nationkey", "l_extendedprice", "l_discount", "l_shipdate", "o_custkey"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [7, 8, 9, 10, 11] + }, + "hint": { + "outputNames": ["s_nationkey", "l_orderkey", "l_extendedprice", "l_discount", "l_shipdate"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [7, 8] + }, + "hint": { + "outputNames": ["s_suppkey", "s_nationkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["s_suppkey", "s_name", "s_address", "s_nationkey", "s_phone", "s_acctbal", "s_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/supplier.parquet", + "length": "83257", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17, 18, 19, 20] + }, + "hint": { + "outputNames": ["l_orderkey", "l_suppkey", "l_extendedprice", "l_discount", "l_shipdate"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 9131 + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 9861 + } + } + }] + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [9, 10] + }, + "hint": { + "outputNames": ["o_orderkey", "o_custkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders.parquet", + "length": "6003201", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [8, 9] + }, + "hint": { + "outputNames": ["c_custkey", "c_nationkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer.parquet", + "length": "1263269", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5] + }, + "hint": { + "outputNames": ["n_nationkey", "n_name"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["n_nationkey", "n_name", "n_regionkey", "n_comment"], + "struct": { + "types": [{ + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/nation.parquet", + "length": "2248", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "FRANCE" + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "GERMANY" + } + } + }] + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5] + }, + "hint": { + "outputNames": ["n_nationkey", "n_name"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["n_nationkey", "n_name", "n_regionkey", "n_comment"], + "struct": { + "types": [{ + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/nation.parquet", + "length": "2248", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "GERMANY" + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "FRANCE" + } + } + }] + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "FRANCE" + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "GERMANY" + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "GERMANY" + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "FRANCE" + } + } + }] + } + } + }] + } + } + }] + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }, { + "scalarFunction": { + "functionReference": 7, + "outputType": { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "enum": "YEAR" + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + }, { + "scalarFunction": { + "functionReference": 8, + "outputType": { + "decimal": { + "scale": 4, + "precision": 32, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 9, + "outputType": { + "decimal": { + "scale": 2, + "precision": 16, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "literal": { + "decimal": { + "value": "AQAAAAAAAAAAAAAAAAAAAA==", + "precision": 1 + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 10, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 4, + "precision": 38, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "names": ["supp_nation", "cust_nation", "l_year", "revenue"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 4, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 3, + "urn": "extension:io.substrait:functions_datetime" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q08-spark_outcome.txt b/substrait_consumer/snapshots/producer/integration/tpch/q08-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q08-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q08-spark_plan.json b/substrait_consumer/snapshots/producer/integration/tpch/q08-spark_plan.json new file mode 100644 index 00000000..44546757 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q08-spark_plan.json @@ -0,0 +1,2517 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 4, + "uri": "/functions_arithmetic_decimal.yaml" + }, { + "extensionUriAnchor": 3, + "uri": "/functions_datetime.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 3, + "name": "equal:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 4, + "name": "gte:date_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 5, + "name": "lte:date_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 6, + "name": "extract:req_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 7, + "name": "multiply:dec_dec", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 8, + "name": "subtract:dec_dec", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 9, + "name": "sum:dec", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 10, + "name": "divide:dec_dec", + "extensionUrnReference": 4 + } + }], + "relations": [{ + "root": { + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [3, 4] + }, + "hint": { + "outputNames": ["o_year", "mkt_share"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [6, 7, 8] + }, + "hint": { + "outputNames": ["o_year", "volume", "nation"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [7, 8, 9, 10, 11] + }, + "hint": { + "outputNames": ["l_extendedprice", "l_discount", "o_orderdate", "n_regionkey", "n_name"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [7, 8, 9, 10, 11] + }, + "hint": { + "outputNames": ["l_extendedprice", "l_discount", "s_nationkey", "o_orderdate", "n_regionkey"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [7, 8, 9, 10, 11] + }, + "hint": { + "outputNames": ["l_extendedprice", "l_discount", "s_nationkey", "o_orderdate", "c_nationkey"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [7, 8, 9, 10, 11] + }, + "hint": { + "outputNames": ["l_extendedprice", "l_discount", "s_nationkey", "o_custkey", "o_orderdate"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [6, 7, 8, 9] + }, + "hint": { + "outputNames": ["l_orderkey", "l_extendedprice", "l_discount", "s_nationkey"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [6, 7, 8, 9] + }, + "hint": { + "outputNames": ["l_orderkey", "l_suppkey", "l_extendedprice", "l_discount"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [9] + }, + "hint": { + "outputNames": ["p_partkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["p_partkey", "p_name", "p_mfgr", "p_brand", "p_type", "p_size", "p_container", "p_retailprice", "p_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/part.parquet", + "length": "695574", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "ECONOMY ANODIZED STEEL" + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17, 18, 19, 20] + }, + "hint": { + "outputNames": ["l_orderkey", "l_partkey", "l_suppkey", "l_extendedprice", "l_discount"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [7, 8] + }, + "hint": { + "outputNames": ["s_suppkey", "s_nationkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["s_suppkey", "s_name", "s_address", "s_nationkey", "s_phone", "s_acctbal", "s_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/supplier.parquet", + "length": "83257", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [9, 10, 11] + }, + "hint": { + "outputNames": ["o_orderkey", "o_custkey", "o_orderdate"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders.parquet", + "length": "6003201", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 9131 + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 9861 + } + } + }] + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [8, 9] + }, + "hint": { + "outputNames": ["c_custkey", "c_nationkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer.parquet", + "length": "1263269", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5] + }, + "hint": { + "outputNames": ["n_nationkey", "n_regionkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["n_nationkey", "n_name", "n_regionkey", "n_comment"], + "struct": { + "types": [{ + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/nation.parquet", + "length": "2248", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5] + }, + "hint": { + "outputNames": ["n_nationkey", "n_name"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["n_nationkey", "n_name", "n_regionkey", "n_comment"], + "struct": { + "types": [{ + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/nation.parquet", + "length": "2248", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [3] + }, + "hint": { + "outputNames": ["r_regionkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["r_regionkey", "r_name", "r_comment"], + "struct": { + "types": [{ + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/region.parquet", + "length": "1058", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "AMERICA" + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "scalarFunction": { + "functionReference": 6, + "outputType": { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "enum": "YEAR" + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + }, { + "scalarFunction": { + "functionReference": 7, + "outputType": { + "decimal": { + "scale": 4, + "precision": 32, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 8, + "outputType": { + "decimal": { + "scale": 2, + "precision": 16, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "literal": { + "decimal": { + "value": "AQAAAAAAAAAAAAAAAAAAAA==", + "precision": 1 + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 9, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 4, + "precision": 38, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "ifThen": { + "ifs": [{ + "if": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "BRAZIL" + } + } + }] + } + }, + "then": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }], + "else": { + "literal": { + "decimal": { + "value": "AAAAAAAAAAAAAAAAAAAAAA==", + "precision": 32, + "scale": 4 + } + } + } + } + } + }] + } + }, { + "measure": { + "functionReference": 9, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 4, + "precision": 38, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "scalarFunction": { + "functionReference": 10, + "outputType": { + "decimal": { + "scale": 6, + "precision": 38, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "names": ["o_year", "mkt_share"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 4, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 3, + "urn": "extension:io.substrait:functions_datetime" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q09-spark_outcome.txt b/substrait_consumer/snapshots/producer/integration/tpch/q09-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q09-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q09-spark_plan.json b/substrait_consumer/snapshots/producer/integration/tpch/q09-spark_plan.json new file mode 100644 index 00000000..8845df0f --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q09-spark_plan.json @@ -0,0 +1,1944 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 3, + "uri": "/functions_string.yaml" + }, { + "extensionUriAnchor": 5, + "uri": "/functions_arithmetic_decimal.yaml" + }, { + "extensionUriAnchor": 4, + "uri": "/functions_datetime.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 3, + "name": "contains:str_str", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 4, + "name": "equal:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 5, + "name": "extract:req_date", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 5, + "functionAnchor": 6, + "name": "subtract:dec_dec", + "extensionUrnReference": 5 + } + }, { + "extensionFunction": { + "extensionUriReference": 5, + "functionAnchor": 7, + "name": "multiply:dec_dec", + "extensionUrnReference": 5 + } + }, { + "extensionFunction": { + "extensionUriReference": 5, + "functionAnchor": 8, + "name": "sum:dec", + "extensionUrnReference": 5 + } + }], + "relations": [{ + "root": { + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [3, 4, 5] + }, + "hint": { + "outputNames": ["nation", "o_year", "sum_profit"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [8, 9, 10] + }, + "hint": { + "outputNames": ["nation", "o_year", "amount"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [8, 9, 10, 11, 12, 13] + }, + "hint": { + "outputNames": ["l_quantity", "l_extendedprice", "l_discount", "s_nationkey", "ps_supplycost", "o_orderdate"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [10, 11, 12, 13, 14, 15] + }, + "hint": { + "outputNames": ["l_orderkey", "l_quantity", "l_extendedprice", "l_discount", "s_nationkey", "ps_supplycost"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [8, 9, 10, 11, 12, 13, 14] + }, + "hint": { + "outputNames": ["l_orderkey", "l_partkey", "l_suppkey", "l_quantity", "l_extendedprice", "l_discount", "s_nationkey"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [7, 8, 9, 10, 11, 12] + }, + "hint": { + "outputNames": ["l_orderkey", "l_partkey", "l_suppkey", "l_quantity", "l_extendedprice", "l_discount"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [9] + }, + "hint": { + "outputNames": ["p_partkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["p_partkey", "p_name", "p_mfgr", "p_brand", "p_type", "p_size", "p_container", "p_retailprice", "p_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/part.parquet", + "length": "695574", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "green" + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17, 18, 19, 20, 21] + }, + "hint": { + "outputNames": ["l_orderkey", "l_partkey", "l_suppkey", "l_quantity", "l_extendedprice", "l_discount"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [7, 8] + }, + "hint": { + "outputNames": ["s_suppkey", "s_nationkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["s_suppkey", "s_name", "s_address", "s_nationkey", "s_phone", "s_acctbal", "s_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/supplier.parquet", + "length": "83257", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6, 7] + }, + "hint": { + "outputNames": ["ps_partkey", "ps_suppkey", "ps_supplycost"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 8 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 9 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [9, 10] + }, + "hint": { + "outputNames": ["o_orderkey", "o_orderdate"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders.parquet", + "length": "6003201", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5] + }, + "hint": { + "outputNames": ["n_nationkey", "n_name"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["n_nationkey", "n_name", "n_regionkey", "n_comment"], + "struct": { + "types": [{ + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/nation.parquet", + "length": "2248", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }, { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "enum": "YEAR" + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }] + } + }, { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "decimal": { + "scale": 4, + "precision": 33, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 7, + "outputType": { + "decimal": { + "scale": 4, + "precision": 32, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "decimal": { + "scale": 2, + "precision": 16, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "literal": { + "decimal": { + "value": "AQAAAAAAAAAAAAAAAAAAAA==", + "precision": 1 + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 7, + "outputType": { + "decimal": { + "scale": 4, + "precision": 31, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 8, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 4, + "precision": 38, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_DESC_NULLS_LAST" + }] + } + }, + "names": ["nation", "o_year", "sum_profit"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 5, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 4, + "urn": "extension:io.substrait:functions_datetime" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }, { + "extensionUrnAnchor": 3, + "urn": "extension:io.substrait:functions_string" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q10-spark_outcome.txt b/substrait_consumer/snapshots/producer/integration/tpch/q10-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q10-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q10-spark_plan.json b/substrait_consumer/snapshots/producer/integration/tpch/q10-spark_plan.json new file mode 100644 index 00000000..9854cfd7 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q10-spark_plan.json @@ -0,0 +1,1631 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 4, + "uri": "/functions_arithmetic_decimal.yaml" + }, { + "extensionUriAnchor": 3, + "uri": "/functions_datetime.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 3, + "name": "gte:date_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 4, + "name": "lt:date_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 5, + "name": "equal:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 6, + "name": "multiply:dec_dec", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 7, + "name": "subtract:dec_dec", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 8, + "name": "sum:dec", + "extensionUrnReference": 4 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [8, 9, 10, 11, 12, 13, 14, 15] + }, + "hint": { + "outputNames": ["c_custkey", "c_name", "revenue", "c_acctbal", "n_name", "c_address", "c_phone", "c_comment"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [11, 12, 13, 14, 15, 16, 17, 18, 19] + }, + "hint": { + "outputNames": ["c_custkey", "c_name", "c_address", "c_phone", "c_acctbal", "c_comment", "l_extendedprice", "l_discount", "n_name"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [11, 12, 13, 14, 15, 16, 17, 18, 19] + }, + "hint": { + "outputNames": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_comment", "l_extendedprice", "l_discount"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [9, 10, 11, 12, 13, 14, 15, 16] + }, + "hint": { + "outputNames": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_comment", "o_orderkey"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [8, 9, 10, 11, 12, 13, 14] + }, + "hint": { + "outputNames": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_comment"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer.parquet", + "length": "1263269", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [9, 10] + }, + "hint": { + "outputNames": ["o_orderkey", "o_custkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders.parquet", + "length": "6003201", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 8674 + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 8766 + } + } + }] + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 8 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17, 18] + }, + "hint": { + "outputNames": ["l_orderkey", "l_extendedprice", "l_discount"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 8 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 8 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "R" + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 8 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 9 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5] + }, + "hint": { + "outputNames": ["n_nationkey", "n_name"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["n_nationkey", "n_name", "n_regionkey", "n_comment"], + "struct": { + "types": [{ + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/nation.parquet", + "length": "2248", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 9 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 8 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 8 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 8, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 4, + "precision": 38, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "decimal": { + "scale": 4, + "precision": 32, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 7, + "outputType": { + "decimal": { + "scale": 2, + "precision": 16, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "literal": { + "decimal": { + "value": "AQAAAAAAAAAAAAAAAAAAAA==", + "precision": 1 + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_DESC_NULLS_LAST" + }] + } + }, + "offset": "0", + "count": "20" + } + }, + "names": ["c_custkey", "c_name", "revenue", "c_acctbal", "n_name", "c_address", "c_phone", "c_comment"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 4, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 3, + "urn": "extension:io.substrait:functions_datetime" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q11-spark_outcome.txt b/substrait_consumer/snapshots/producer/integration/tpch/q11-spark_outcome.txt new file mode 100644 index 00000000..ecd3b783 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q11-spark_outcome.txt @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q12-spark_outcome.txt b/substrait_consumer/snapshots/producer/integration/tpch/q12-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q12-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q12-spark_plan.json b/substrait_consumer/snapshots/producer/integration/tpch/q12-spark_plan.json new file mode 100644 index 00000000..2ec3ff70 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q12-spark_plan.json @@ -0,0 +1,1080 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 4, + "uri": "/functions_arithmetic.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 3, + "uri": "/functions_datetime.yaml" + }, { + "extensionUriAnchor": 1, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "is_not_null:any", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "and:bool", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 3, + "name": "lt:date_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 4, + "name": "gte:date_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 5, + "name": "equal:any_any", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 6, + "name": "or:bool", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 7, + "name": "sum:i32", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 8, + "name": "not:bool", + "extensionUrnReference": 2 + } + }], + "relations": [{ + "root": { + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [3, 4, 5] + }, + "hint": { + "outputNames": ["l_shipmode", "high_line_count", "low_line_count"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5] + }, + "hint": { + "outputNames": ["o_orderpriority", "l_shipmode"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [9, 10] + }, + "hint": { + "outputNames": ["o_orderkey", "o_orderpriority"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders.parquet", + "length": "6003201", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17] + }, + "hint": { + "outputNames": ["l_orderkey", "l_shipmode"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 11 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 12 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "singularOrList": { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 14 + } + }, + "rootReference": { + } + } + }, + "options": [{ + "literal": { + "string": "MAIL" + } + }, { + "literal": { + "string": "SHIP" + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 11 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 12 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 11 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 12 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 8766 + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 12 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 9131 + } + } + }] + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 14 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 7, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "ifThen": { + "ifs": [{ + "if": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "1-URGENT" + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "2-HIGH" + } + } + }] + } + } + }] + } + }, + "then": { + "literal": { + "i32": 1 + } + } + }], + "else": { + "literal": { + "i32": 0 + } + } + } + } + }] + } + }, { + "measure": { + "functionReference": 7, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "ifThen": { + "ifs": [{ + "if": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 8, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "1-URGENT" + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 8, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "2-HIGH" + } + } + }] + } + } + }] + } + } + }] + } + }, + "then": { + "literal": { + "i32": 1 + } + } + }], + "else": { + "literal": { + "i32": 0 + } + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "names": ["l_shipmode", "high_line_count", "low_line_count"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 3, + "urn": "extension:io.substrait:functions_datetime" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_boolean" + }, { + "extensionUrnAnchor": 4, + "urn": "extension:io.substrait:functions_arithmetic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q13-spark_outcome.txt b/substrait_consumer/snapshots/producer/integration/tpch/q13-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q13-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q13-spark_plan.json b/substrait_consumer/snapshots/producer/integration/tpch/q13-spark_plan.json new file mode 100644 index 00000000..2bb519a8 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q13-spark_plan.json @@ -0,0 +1,621 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 4, + "uri": "/functions_aggregate_generic.yaml" + }, { + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 3, + "uri": "/functions_string.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 3, + "name": "not:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 4, + "name": "like:str_str", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 5, + "name": "equal:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 6, + "name": "count:any", + "extensionUrnReference": 4 + } + }], + "relations": [{ + "root": { + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [2, 3] + }, + "hint": { + "outputNames": ["c_count", "custdist"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [2] + }, + "hint": { + "outputNames": ["c_count"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [3, 4] + }, + "hint": { + "outputNames": ["c_custkey", "o_orderkey"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [8] + }, + "hint": { + "outputNames": ["c_custkey"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer.parquet", + "length": "1263269", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [9, 10] + }, + "hint": { + "outputNames": ["o_orderkey", "o_custkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders.parquet", + "length": "6003201", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 8 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 8 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "%special%requests%" + } + } + }] + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_LEFT" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 6, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "i64": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 6, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "i64": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "literal": { + "i32": 1 + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_DESC_NULLS_LAST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_DESC_NULLS_LAST" + }] + } + }, + "names": ["c_count", "custdist"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }, { + "extensionUrnAnchor": 4, + "urn": "extension:io.substrait:functions_aggregate_generic" + }, { + "extensionUrnAnchor": 3, + "urn": "extension:io.substrait:functions_string" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q14-spark_outcome.txt b/substrait_consumer/snapshots/producer/integration/tpch/q14-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q14-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q14-spark_plan.json b/substrait_consumer/snapshots/producer/integration/tpch/q14-spark_plan.json new file mode 100644 index 00000000..5cf7b18e --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q14-spark_plan.json @@ -0,0 +1,887 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 4, + "uri": "/functions_string.yaml" + }, { + "extensionUriAnchor": 5, + "uri": "/functions_arithmetic_decimal.yaml" + }, { + "extensionUriAnchor": 3, + "uri": "/functions_datetime.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 3, + "name": "gte:date_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 4, + "name": "lt:date_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 5, + "name": "equal:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 6, + "name": "starts_with:str_str", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 5, + "functionAnchor": 7, + "name": "multiply:dec_dec", + "extensionUrnReference": 5 + } + }, { + "extensionFunction": { + "extensionUriReference": 5, + "functionAnchor": 8, + "name": "subtract:dec_dec", + "extensionUrnReference": 5 + } + }, { + "extensionFunction": { + "extensionUriReference": 5, + "functionAnchor": 9, + "name": "sum:dec", + "extensionUrnReference": 5 + } + }, { + "extensionFunction": { + "extensionUriReference": 5, + "functionAnchor": 10, + "name": "divide:dec_dec", + "extensionUrnReference": 5 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [2] + }, + "hint": { + "outputNames": ["promo_revenue"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6, 7] + }, + "hint": { + "outputNames": ["l_extendedprice", "l_discount", "p_type"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17, 18] + }, + "hint": { + "outputNames": ["l_partkey", "l_extendedprice", "l_discount"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 9374 + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 9404 + } + } + }] + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [9, 10] + }, + "hint": { + "outputNames": ["p_partkey", "p_type"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["p_partkey", "p_name", "p_mfgr", "p_brand", "p_type", "p_size", "p_container", "p_retailprice", "p_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/part.parquet", + "length": "695574", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + }], + "measures": [{ + "measure": { + "functionReference": 9, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 4, + "precision": 38, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "ifThen": { + "ifs": [{ + "if": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "PROMO" + } + } + }] + } + }, + "then": { + "scalarFunction": { + "functionReference": 7, + "outputType": { + "decimal": { + "scale": 4, + "precision": 32, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 8, + "outputType": { + "decimal": { + "scale": 2, + "precision": 16, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "literal": { + "decimal": { + "value": "AQAAAAAAAAAAAAAAAAAAAA==", + "precision": 1 + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }], + "else": { + "literal": { + "decimal": { + "value": "AAAAAAAAAAAAAAAAAAAAAA==", + "precision": 32, + "scale": 4 + } + } + } + } + } + }] + } + }, { + "measure": { + "functionReference": 9, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 4, + "precision": 38, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 7, + "outputType": { + "decimal": { + "scale": 4, + "precision": 32, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 8, + "outputType": { + "decimal": { + "scale": 2, + "precision": 16, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "literal": { + "decimal": { + "value": "AQAAAAAAAAAAAAAAAAAAAA==", + "precision": 1 + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }] + } + }] + } + }, + "expressions": [{ + "scalarFunction": { + "functionReference": 10, + "outputType": { + "decimal": { + "scale": 6, + "precision": 38, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 7, + "outputType": { + "decimal": { + "scale": 6, + "precision": 38, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "literal": { + "decimal": { + "value": "ECcAAAAAAAAAAAAAAAAAAA==", + "precision": 5, + "scale": 2 + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "names": ["promo_revenue"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 5, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 3, + "urn": "extension:io.substrait:functions_datetime" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }, { + "extensionUrnAnchor": 4, + "urn": "extension:io.substrait:functions_string" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q15-spark_outcome.txt b/substrait_consumer/snapshots/producer/integration/tpch/q15-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q15-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q15-spark_plan.json b/substrait_consumer/snapshots/producer/integration/tpch/q15-spark_plan.json new file mode 100644 index 00000000..0733ac66 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q15-spark_plan.json @@ -0,0 +1,1283 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 2, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 4, + "uri": "/functions_arithmetic_decimal.yaml" + }, { + "extensionUriAnchor": 3, + "uri": "/functions_datetime.yaml" + }, { + "extensionUriAnchor": 1, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "is_not_null:any", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "and:bool", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 3, + "name": "gte:date_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 4, + "name": "lt:date_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 5, + "name": "multiply:dec_dec", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 6, + "name": "subtract:dec_dec", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 7, + "name": "sum:dec", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 8, + "name": "equal:any_any", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 9, + "name": "max:dec", + "extensionUrnReference": 4 + } + }], + "relations": [{ + "root": { + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [6, 7, 8, 9, 10] + }, + "hint": { + "outputNames": ["s_suppkey", "s_name", "s_address", "s_phone", "total_revenue"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [7, 8, 9, 10] + }, + "hint": { + "outputNames": ["s_suppkey", "s_name", "s_address", "s_phone"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["s_suppkey", "s_name", "s_address", "s_nationkey", "s_phone", "s_acctbal", "s_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/supplier.parquet", + "length": "83257", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [2, 3] + }, + "hint": { + "outputNames": ["supplier_no", "total_revenue"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17, 18] + }, + "hint": { + "outputNames": ["l_suppkey", "l_extendedprice", "l_discount"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 9496 + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 9587 + } + } + }] + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 7, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 4, + "precision": 38, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "decimal": { + "scale": 4, + "precision": 32, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "decimal": { + "scale": 2, + "precision": 16, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "literal": { + "decimal": { + "value": "AQAAAAAAAAAAAAAAAAAAAA==", + "precision": 1 + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "condition": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 8, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "subquery": { + "scalar": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [1] + }, + "hint": { + "outputNames": ["max(total_revenue)"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [2] + }, + "hint": { + "outputNames": ["total_revenue"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17, 18] + }, + "hint": { + "outputNames": ["l_suppkey", "l_extendedprice", "l_discount"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 9496 + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 9587 + } + } + }] + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 7, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 4, + "precision": 38, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "decimal": { + "scale": 4, + "precision": 32, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "decimal": { + "scale": 2, + "precision": 16, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "literal": { + "decimal": { + "value": "AQAAAAAAAAAAAAAAAAAAAA==", + "precision": 1 + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + }], + "measures": [{ + "measure": { + "functionReference": 9, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 4, + "precision": 38, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + } + } + } + } + }] + } + } + }] + } + } + } + }, + "expression": { + "scalarFunction": { + "functionReference": 8, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "names": ["s_suppkey", "s_name", "s_address", "s_phone", "total_revenue"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 4, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 3, + "urn": "extension:io.substrait:functions_datetime" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_boolean" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q16-spark_outcome.txt b/substrait_consumer/snapshots/producer/integration/tpch/q16-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q16-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q16-spark_plan.json b/substrait_consumer/snapshots/producer/integration/tpch/q16-spark_plan.json new file mode 100644 index 00000000..5d1bdb7f --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q16-spark_plan.json @@ -0,0 +1,1117 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 4, + "uri": "/functions_aggregate_generic.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 3, + "uri": "/functions_string.yaml" + }, { + "extensionUriAnchor": 1, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "is_not_null:any", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "and:bool", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 3, + "name": "like:str_str", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 4, + "name": "or:bool", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 5, + "name": "equal:any_any", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 6, + "name": "is_null:any", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 7, + "name": "not:bool", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 8, + "name": "starts_with:str_str", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 9, + "name": "count:any", + "extensionUrnReference": 4 + } + }], + "relations": [{ + "root": { + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5, 6, 7] + }, + "hint": { + "outputNames": ["p_brand", "p_type", "p_size", "supplier_cnt"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [6, 7, 8, 9] + }, + "hint": { + "outputNames": ["ps_suppkey", "p_brand", "p_type", "p_size"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6] + }, + "hint": { + "outputNames": ["ps_partkey", "ps_suppkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [7] + }, + "hint": { + "outputNames": ["s_suppkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["s_suppkey", "s_name", "s_address", "s_nationkey", "s_phone", "s_acctbal", "s_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/supplier.parquet", + "length": "83257", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "%Customer%Complaints%" + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }] + } + }, + "type": "JOIN_TYPE_LEFT_ANTI" + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [9, 10, 11, 12] + }, + "hint": { + "outputNames": ["p_partkey", "p_brand", "p_type", "p_size"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["p_partkey", "p_name", "p_mfgr", "p_brand", "p_type", "p_size", "p_container", "p_retailprice", "p_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/part.parquet", + "length": "695574", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 7, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "Brand#45" + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 7, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 8, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "MEDIUM POLISHED" + } + } + }] + } + } + }] + } + } + }] + } + } + }, { + "value": { + "singularOrList": { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, + "options": [{ + "literal": { + "i32": 49 + } + }, { + "literal": { + "i32": 14 + } + }, { + "literal": { + "i32": 23 + } + }, { + "literal": { + "i32": 45 + } + }, { + "literal": { + "i32": 19 + } + }, { + "literal": { + "i32": 3 + } + }, { + "literal": { + "i32": 36 + } + }, { + "literal": { + "i32": 9 + } + }] + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 9, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "i64": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "invocation": "AGGREGATION_INVOCATION_DISTINCT", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_DESC_NULLS_LAST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "names": ["p_brand", "p_type", "p_size", "supplier_cnt"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_boolean" + }, { + "extensionUrnAnchor": 4, + "urn": "extension:io.substrait:functions_aggregate_generic" + }, { + "extensionUrnAnchor": 3, + "urn": "extension:io.substrait:functions_string" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q17-spark_outcome.txt b/substrait_consumer/snapshots/producer/integration/tpch/q17-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q17-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q17-spark_plan.json b/substrait_consumer/snapshots/producer/integration/tpch/q17-spark_plan.json new file mode 100644 index 00000000..0e34abbb --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q17-spark_plan.json @@ -0,0 +1,1148 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 3, + "uri": "/functions_arithmetic_decimal.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 3, + "name": "equal:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 4, + "name": "avg:dec", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 5, + "name": "multiply:dec_dec", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 6, + "name": "lt:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 7, + "name": "sum:dec", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 8, + "name": "divide:dec_dec", + "extensionUrnReference": 3 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [1] + }, + "hint": { + "outputNames": ["avg_yearly"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5] + }, + "hint": { + "outputNames": ["l_extendedprice"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5, 6] + }, + "hint": { + "outputNames": ["l_quantity", "l_extendedprice", "p_partkey"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17, 18] + }, + "hint": { + "outputNames": ["l_partkey", "l_quantity", "l_extendedprice"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [9] + }, + "hint": { + "outputNames": ["p_partkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["p_partkey", "p_name", "p_mfgr", "p_brand", "p_type", "p_size", "p_container", "p_retailprice", "p_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/part.parquet", + "length": "695574", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "Brand#23" + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "MED BOX" + } + } + }] + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [2, 3] + }, + "hint": { + "outputNames": ["(0.2 * avg(l_quantity))", "l_partkey"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17] + }, + "hint": { + "outputNames": ["l_partkey", "l_quantity"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 4, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 6, + "precision": 19, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "scalarFunction": { + "functionReference": 5, + "outputType": { + "decimal": { + "scale": 7, + "precision": 21, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "literal": { + "decimal": { + "value": "AgAAAAAAAAAAAAAAAAAAAA==", + "precision": 1, + "scale": 1 + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + }, { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "condition": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + } + }, + "expression": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "cast": { + "type": { + "decimal": { + "scale": 7, + "precision": 21, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "input": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "failureBehavior": "FAILURE_BEHAVIOR_THROW_EXCEPTION" + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + }], + "measures": [{ + "measure": { + "functionReference": 7, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 2, + "precision": 25, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "scalarFunction": { + "functionReference": 8, + "outputType": { + "decimal": { + "scale": 6, + "precision": 30, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "decimal": { + "value": "RgAAAAAAAAAAAAAAAAAAAA==", + "precision": 2, + "scale": 1 + } + } + } + }] + } + }] + } + }, + "names": ["avg_yearly"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 3, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q18-spark_outcome.txt b/substrait_consumer/snapshots/producer/integration/tpch/q18-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q18-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q18-spark_plan.json b/substrait_consumer/snapshots/producer/integration/tpch/q18-spark_plan.json new file mode 100644 index 00000000..2afccd01 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q18-spark_plan.json @@ -0,0 +1,1643 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 2, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 3, + "uri": "/functions_arithmetic_decimal.yaml" + }, { + "extensionUriAnchor": 1, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "is_not_null:any", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "and:bool", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 3, + "name": "sum:dec", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 4, + "name": "gt:any_any", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 5, + "name": "equal:any_any", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [6, 7, 8, 9, 10, 11] + }, + "hint": { + "outputNames": ["c_name", "c_custkey", "o_orderkey", "o_orderdate", "o_totalprice", "sum(l_quantity)"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [7, 8, 9, 10, 11, 12] + }, + "hint": { + "outputNames": ["c_custkey", "c_name", "o_orderkey", "o_totalprice", "o_orderdate", "l_quantity"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [6, 7, 8, 9, 10] + }, + "hint": { + "outputNames": ["c_custkey", "c_name", "o_orderkey", "o_totalprice", "o_orderdate"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [8, 9] + }, + "hint": { + "outputNames": ["c_custkey", "c_name"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer.parquet", + "length": "1263269", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [9, 10, 11, 12] + }, + "hint": { + "outputNames": ["o_orderkey", "o_custkey", "o_totalprice", "o_orderdate"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders.parquet", + "length": "6003201", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [2] + }, + "hint": { + "outputNames": ["l_orderkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [2, 3] + }, + "hint": { + "outputNames": ["l_orderkey", "sum(l_quantity#2196)"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17] + }, + "hint": { + "outputNames": ["l_orderkey", "l_quantity"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 3, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 2, + "precision": 25, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "condition": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "decimal": { + "value": "MHUAAAAAAAAAAAAAAAAAAA==", + "precision": 25, + "scale": 2 + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_LEFT_SEMI" + } + }, + "expression": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17] + }, + "hint": { + "outputNames": ["l_orderkey", "l_quantity"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [2] + }, + "hint": { + "outputNames": ["l_orderkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [2, 3] + }, + "hint": { + "outputNames": ["l_orderkey", "sum(l_quantity#2196)"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17] + }, + "hint": { + "outputNames": ["l_orderkey", "l_quantity"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 3, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 2, + "precision": 25, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "condition": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "decimal": { + "value": "MHUAAAAAAAAAAAAAAAAAAA==", + "precision": 25, + "scale": 2 + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_LEFT_SEMI" + } + }, + "expression": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 3, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 2, + "precision": 25, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_DESC_NULLS_LAST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "offset": "0", + "count": "100" + } + }, + "names": ["c_name", "c_custkey", "o_orderkey", "o_orderdate", "o_totalprice", "sum(l_quantity)"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 3, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_boolean" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q19-spark_outcome.txt b/substrait_consumer/snapshots/producer/integration/tpch/q19-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q19-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q19-spark_plan.json b/substrait_consumer/snapshots/producer/integration/tpch/q19-spark_plan.json new file mode 100644 index 00000000..94c9078d --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q19-spark_plan.json @@ -0,0 +1,2119 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 3, + "uri": "/functions_arithmetic_decimal.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 3, + "name": "equal:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 4, + "name": "or:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 5, + "name": "gte:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 6, + "name": "lte:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 7, + "name": "multiply:dec_dec", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 8, + "name": "subtract:dec_dec", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 9, + "name": "sum:dec", + "extensionUrnReference": 3 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [1] + }, + "hint": { + "outputNames": ["revenue"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [8, 9] + }, + "hint": { + "outputNames": ["l_extendedprice", "l_discount"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17, 18, 19] + }, + "hint": { + "outputNames": ["l_partkey", "l_quantity", "l_extendedprice", "l_discount"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 13 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "singularOrList": { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 14 + } + }, + "rootReference": { + } + } + }, + "options": [{ + "literal": { + "string": "AIR" + } + }, { + "literal": { + "string": "AIR REG" + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 13 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "DELIVER IN PERSON" + } + } + }] + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "decimal": { + "value": "ZAAAAAAAAAAAAAAAAAAAAA==", + "precision": 15, + "scale": 2 + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "decimal": { + "value": "TAQAAAAAAAAAAAAAAAAAAA==", + "precision": 15, + "scale": 2 + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "decimal": { + "value": "6AMAAAAAAAAAAAAAAAAAAA==", + "precision": 15, + "scale": 2 + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "decimal": { + "value": "0AcAAAAAAAAAAAAAAAAAAA==", + "precision": 15, + "scale": 2 + } + } + } + }] + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "decimal": { + "value": "0AcAAAAAAAAAAAAAAAAAAA==", + "precision": 15, + "scale": 2 + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "decimal": { + "value": "uAsAAAAAAAAAAAAAAAAAAA==", + "precision": 15, + "scale": 2 + } + } + } + }] + } + } + }] + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [9, 10, 11, 12] + }, + "hint": { + "outputNames": ["p_partkey", "p_brand", "p_size", "p_container"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["p_partkey", "p_name", "p_mfgr", "p_brand", "p_type", "p_size", "p_container", "p_retailprice", "p_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/part.parquet", + "length": "695574", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "i32": 1 + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "Brand#12" + } + } + }] + } + } + }, { + "value": { + "singularOrList": { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }, + "options": [{ + "literal": { + "string": "SM CASE" + } + }, { + "literal": { + "string": "SM BOX" + } + }, { + "literal": { + "string": "SM PACK" + } + }, { + "literal": { + "string": "SM PKG" + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "i32": 5 + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "Brand#23" + } + } + }] + } + } + }, { + "value": { + "singularOrList": { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }, + "options": [{ + "literal": { + "string": "MED BAG" + } + }, { + "literal": { + "string": "MED BOX" + } + }, { + "literal": { + "string": "MED PKG" + } + }, { + "literal": { + "string": "MED PACK" + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "i32": 10 + } + } + }] + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "Brand#34" + } + } + }] + } + } + }, { + "value": { + "singularOrList": { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }, + "options": [{ + "literal": { + "string": "LG CASE" + } + }, { + "literal": { + "string": "LG BOX" + } + }, { + "literal": { + "string": "LG PACK" + } + }, { + "literal": { + "string": "LG PKG" + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "i32": 15 + } + } + }] + } + } + }] + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "Brand#12" + } + } + }] + } + } + }, { + "value": { + "singularOrList": { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }, + "options": [{ + "literal": { + "string": "SM CASE" + } + }, { + "literal": { + "string": "SM BOX" + } + }, { + "literal": { + "string": "SM PACK" + } + }, { + "literal": { + "string": "SM PKG" + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "decimal": { + "value": "ZAAAAAAAAAAAAAAAAAAAAA==", + "precision": 15, + "scale": 2 + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "decimal": { + "value": "TAQAAAAAAAAAAAAAAAAAAA==", + "precision": 15, + "scale": 2 + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "i32": 5 + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "Brand#23" + } + } + }] + } + } + }, { + "value": { + "singularOrList": { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }, + "options": [{ + "literal": { + "string": "MED BAG" + } + }, { + "literal": { + "string": "MED BOX" + } + }, { + "literal": { + "string": "MED PKG" + } + }, { + "literal": { + "string": "MED PACK" + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "decimal": { + "value": "6AMAAAAAAAAAAAAAAAAAAA==", + "precision": 15, + "scale": 2 + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "decimal": { + "value": "0AcAAAAAAAAAAAAAAAAAAA==", + "precision": 15, + "scale": 2 + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "i32": 10 + } + } + }] + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "Brand#34" + } + } + }] + } + } + }, { + "value": { + "singularOrList": { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }, + "options": [{ + "literal": { + "string": "LG CASE" + } + }, { + "literal": { + "string": "LG BOX" + } + }, { + "literal": { + "string": "LG PACK" + } + }, { + "literal": { + "string": "LG PKG" + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "decimal": { + "value": "0AcAAAAAAAAAAAAAAAAAAA==", + "precision": 15, + "scale": 2 + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "decimal": { + "value": "uAsAAAAAAAAAAAAAAAAAAA==", + "precision": 15, + "scale": 2 + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "i32": 15 + } + } + }] + } + } + }] + } + } + }] + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + }], + "measures": [{ + "measure": { + "functionReference": 9, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 4, + "precision": 38, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 7, + "outputType": { + "decimal": { + "scale": 4, + "precision": 32, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 8, + "outputType": { + "decimal": { + "scale": 2, + "precision": 16, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "literal": { + "decimal": { + "value": "AQAAAAAAAAAAAAAAAAAAAA==", + "precision": 1 + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["revenue"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 3, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q20-spark_outcome.txt b/substrait_consumer/snapshots/producer/integration/tpch/q20-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q20-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q20-spark_plan.json b/substrait_consumer/snapshots/producer/integration/tpch/q20-spark_plan.json new file mode 100644 index 00000000..3e0a4f5c --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q20-spark_plan.json @@ -0,0 +1,1834 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 2, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 3, + "uri": "/functions_string.yaml" + }, { + "extensionUriAnchor": 5, + "uri": "/functions_arithmetic_decimal.yaml" + }, { + "extensionUriAnchor": 4, + "uri": "/functions_datetime.yaml" + }, { + "extensionUriAnchor": 1, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "is_not_null:any", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "and:bool", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 3, + "name": "starts_with:str_str", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 4, + "name": "equal:any_any", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 5, + "name": "gte:date_date", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 6, + "name": "lt:date_date", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 5, + "functionAnchor": 7, + "name": "sum:dec", + "extensionUrnReference": 5 + } + }, { + "extensionFunction": { + "extensionUriReference": 5, + "functionAnchor": 8, + "name": "multiply:dec_dec", + "extensionUrnReference": 5 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 9, + "name": "gt:any_any", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5] + }, + "hint": { + "outputNames": ["s_name", "s_address"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5, 6] + }, + "hint": { + "outputNames": ["s_name", "s_address", "s_nationkey"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [7, 8, 9, 10] + }, + "hint": { + "outputNames": ["s_suppkey", "s_name", "s_address", "s_nationkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["s_suppkey", "s_name", "s_address", "s_nationkey", "s_phone", "s_acctbal", "s_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/supplier.parquet", + "length": "83257", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [6] + }, + "hint": { + "outputNames": ["ps_suppkey"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6, 7] + }, + "hint": { + "outputNames": ["ps_partkey", "ps_suppkey", "ps_availqty"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [9] + }, + "hint": { + "outputNames": ["p_partkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["p_partkey", "p_name", "p_mfgr", "p_brand", "p_type", "p_size", "p_container", "p_retailprice", "p_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/part.parquet", + "length": "695574", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "forest" + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_LEFT_SEMI" + } + }, + "right": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [3, 4, 5] + }, + "hint": { + "outputNames": ["(0.5 * sum(l_quantity))", "l_partkey", "l_suppkey"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17, 18] + }, + "hint": { + "outputNames": ["l_partkey", "l_suppkey", "l_quantity"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 8766 + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 10 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "date": 9131 + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [9] + }, + "hint": { + "outputNames": ["p_partkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["p_partkey", "p_name", "p_mfgr", "p_brand", "p_type", "p_size", "p_container", "p_retailprice", "p_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/part.parquet", + "length": "695574", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "forest" + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_LEFT_SEMI" + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 7, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 2, + "precision": 25, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "scalarFunction": { + "functionReference": 8, + "outputType": { + "decimal": { + "scale": 3, + "precision": 27, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "literal": { + "decimal": { + "value": "BQAAAAAAAAAAAAAAAAAAAA==", + "precision": 1, + "scale": 1 + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + }, { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + } + }, + "expression": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 9, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "cast": { + "type": { + "decimal": { + "scale": 3, + "precision": 27, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "input": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, + "failureBehavior": "FAILURE_BEHAVIOR_THROW_EXCEPTION" + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_LEFT_SEMI" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [4] + }, + "hint": { + "outputNames": ["n_nationkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["n_nationkey", "n_name", "n_regionkey", "n_comment"], + "struct": { + "types": [{ + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/nation.parquet", + "length": "2248", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "CANADA" + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "names": ["s_name", "s_address"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 5, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 4, + "urn": "extension:io.substrait:functions_datetime" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_boolean" + }, { + "extensionUrnAnchor": 3, + "urn": "extension:io.substrait:functions_string" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q21-spark_outcome.txt b/substrait_consumer/snapshots/producer/integration/tpch/q21-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q21-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q21-spark_plan.json b/substrait_consumer/snapshots/producer/integration/tpch/q21-spark_plan.json new file mode 100644 index 00000000..cb2cea34 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q21-spark_plan.json @@ -0,0 +1,1832 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 4, + "uri": "/functions_aggregate_generic.yaml" + }, { + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 3, + "uri": "/functions_datetime.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 3, + "name": "gt:date_date", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 4, + "name": "equal:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 5, + "name": "not:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 6, + "name": "count:any", + "extensionUrnReference": 4 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [2, 3] + }, + "hint": { + "outputNames": ["s_name", "numwait"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [3] + }, + "hint": { + "outputNames": ["s_name"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5] + }, + "hint": { + "outputNames": ["s_name", "s_nationkey"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6, 7] + }, + "hint": { + "outputNames": ["s_name", "s_nationkey", "l_orderkey"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [7, 8, 9] + }, + "hint": { + "outputNames": ["s_suppkey", "s_name", "s_nationkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["s_suppkey", "s_name", "s_address", "s_nationkey", "s_phone", "s_acctbal", "s_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/supplier.parquet", + "length": "83257", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17] + }, + "hint": { + "outputNames": ["l_orderkey", "l_suppkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 12 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 11 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 12 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 11 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17] + }, + "hint": { + "outputNames": ["l_orderkey", "l_suppkey"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }] + } + }, + "type": "JOIN_TYPE_LEFT_SEMI" + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17] + }, + "hint": { + "outputNames": ["l_orderkey", "l_suppkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 12 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 11 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 12 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 11 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 5, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }] + } + }, + "type": "JOIN_TYPE_LEFT_ANTI" + } + }, + "expression": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [9] + }, + "hint": { + "outputNames": ["o_orderkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders.parquet", + "length": "6003201", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "F" + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [4] + }, + "hint": { + "outputNames": ["n_nationkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["n_nationkey", "n_name", "n_regionkey", "n_comment"], + "struct": { + "types": [{ + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/nation.parquet", + "length": "2248", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "SAUDI ARABIA" + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 6, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "i64": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "literal": { + "i32": 1 + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_DESC_NULLS_LAST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "offset": "0", + "count": "100" + } + }, + "names": ["s_name", "numwait"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 3, + "urn": "extension:io.substrait:functions_datetime" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }, { + "extensionUrnAnchor": 4, + "urn": "extension:io.substrait:functions_aggregate_generic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q22-spark_outcome.txt b/substrait_consumer/snapshots/producer/integration/tpch/q22-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q22-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/integration/tpch/q22-spark_plan.json b/substrait_consumer/snapshots/producer/integration/tpch/q22-spark_plan.json new file mode 100644 index 00000000..85057f76 --- /dev/null +++ b/substrait_consumer/snapshots/producer/integration/tpch/q22-spark_plan.json @@ -0,0 +1,974 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 5, + "uri": "/functions_aggregate_generic.yaml" + }, { + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 3, + "uri": "/functions_string.yaml" + }, { + "extensionUriAnchor": 4, + "uri": "/functions_arithmetic_decimal.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 3, + "name": "substring:str_i32_i32", + "extensionUrnReference": 3 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 4, + "name": "gt:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 5, + "name": "avg:dec", + "extensionUrnReference": 4 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 6, + "name": "equal:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 5, + "functionAnchor": 7, + "name": "count:any", + "extensionUrnReference": 5 + } + }, { + "extensionFunction": { + "extensionUriReference": 4, + "functionAnchor": 8, + "name": "sum:dec", + "extensionUrnReference": 4 + } + }], + "relations": [{ + "root": { + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [3, 4, 5] + }, + "hint": { + "outputNames": ["cntrycode", "numcust", "totacctbal"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [3, 4] + }, + "hint": { + "outputNames": ["cntrycode", "c_acctbal"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [8, 9, 10] + }, + "hint": { + "outputNames": ["c_custkey", "c_phone", "c_acctbal"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer.parquet", + "length": "1263269", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "singularOrList": { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "i32": 1 + } + } + }, { + "value": { + "literal": { + "i32": 2 + } + } + }] + } + }, + "options": [{ + "literal": { + "string": "13" + } + }, { + "literal": { + "string": "31" + } + }, { + "literal": { + "string": "23" + } + }, { + "literal": { + "string": "29" + } + }, { + "literal": { + "string": "30" + } + }, { + "literal": { + "string": "18" + } + }, { + "literal": { + "string": "17" + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "cast": { + "type": { + "decimal": { + "scale": 6, + "precision": 19, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "input": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, + "failureBehavior": "FAILURE_BEHAVIOR_THROW_EXCEPTION" + } + } + }, { + "value": { + "subquery": { + "scalar": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [1] + }, + "hint": { + "outputNames": ["avg(c_acctbal)"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [8] + }, + "hint": { + "outputNames": ["c_acctbal"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer.parquet", + "length": "1263269", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "decimal": { + "value": "AAAAAAAAAAAAAAAAAAAAAA==", + "precision": 15, + "scale": 2 + } + } + } + }] + } + } + }, { + "value": { + "singularOrList": { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "i32": 1 + } + } + }, { + "value": { + "literal": { + "i32": 2 + } + } + }] + } + }, + "options": [{ + "literal": { + "string": "13" + } + }, { + "literal": { + "string": "31" + } + }, { + "literal": { + "string": "23" + } + }, { + "literal": { + "string": "29" + } + }, { + "literal": { + "string": "30" + } + }, { + "literal": { + "string": "18" + } + }, { + "literal": { + "string": "17" + } + }] + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + }], + "measures": [{ + "measure": { + "functionReference": 5, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 6, + "precision": 19, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + } + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [9] + }, + "hint": { + "outputNames": ["o_custkey"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders.parquet", + "length": "6003201", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 6, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_LEFT_ANTI" + } + }, + "expressions": [{ + "scalarFunction": { + "functionReference": 3, + "outputType": { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "i32": 1 + } + } + }, { + "value": { + "literal": { + "i32": 2 + } + } + }] + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 7, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "i64": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "literal": { + "i32": 1 + } + } + }] + } + }, { + "measure": { + "functionReference": 8, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 2, + "precision": 25, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "names": ["cntrycode", "numcust", "totacctbal"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 4, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }, { + "extensionUrnAnchor": 5, + "urn": "extension:io.substrait:functions_aggregate_generic" + }, { + "extensionUrnAnchor": 3, + "urn": "extension:io.substrait:functions_string" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_in_subquery-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_in_subquery-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_in_subquery-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_in_subquery-spark_plan.json b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_in_subquery-spark_plan.json new file mode 100644 index 00000000..6fdac286 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_in_subquery-spark_plan.json @@ -0,0 +1,372 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 3, + "uri": "/functions_arithmetic_decimal.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 3, + "name": "lte:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 4, + "name": "avg:dec", + "extensionUrnReference": 3 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [9] + }, + "hint": { + "outputNames": ["O_TOTALPRICE"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders_small.parquet", + "length": "1385", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "cast": { + "type": { + "decimal": { + "scale": 6, + "precision": 19, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "input": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, + "failureBehavior": "FAILURE_BEHAVIOR_THROW_EXCEPTION" + } + } + }, { + "value": { + "subquery": { + "scalar": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [1] + }, + "hint": { + "outputNames": ["avg(O_TOTALPRICE)"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [9] + }, + "hint": { + "outputNames": ["o_totalprice"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders_small.parquet", + "length": "1385", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + }], + "measures": [{ + "measure": { + "functionReference": 4, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 6, + "precision": 19, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + } + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["O_TOTALPRICE"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 3, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }, { + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_computation-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_computation-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_computation-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_computation-spark_plan.json b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_computation-spark_plan.json new file mode 100644 index 00000000..fc26bfb2 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_computation-spark_plan.json @@ -0,0 +1,199 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_arithmetic_decimal.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "avg:dec", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 2, + "name": "multiply:dec_dec", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [1] + }, + "hint": { + "outputNames": ["(avg(O_TOTALPRICE) * 10)"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [9] + }, + "hint": { + "outputNames": ["o_totalprice"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders_small.parquet", + "length": "1385", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + }], + "measures": [{ + "measure": { + "functionReference": 1, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 6, + "precision": 19, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "scalarFunction": { + "functionReference": 2, + "outputType": { + "decimal": { + "scale": 6, + "precision": 22, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "decimal": { + "value": "CgAAAAAAAAAAAAAAAAAAAA==", + "precision": 2 + } + } + } + }] + } + }] + } + }, + "names": ["(avg(O_TOTALPRICE) * 10)"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_group_by-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_group_by-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_group_by-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_group_by-spark_plan.json b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_group_by-spark_plan.json new file mode 100644 index 00000000..d08b2e83 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_group_by-spark_plan.json @@ -0,0 +1,279 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_aggregate_generic.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "count:any", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [3, 4, 5] + }, + "hint": { + "outputNames": ["L_ORDERKEY", "L_LINENUMBER", "count(1)"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17] + }, + "hint": { + "outputNames": ["l_orderkey", "l_linenumber"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem_small.parquet", + "length": "2758", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 1, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "i64": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "literal": { + "i32": 1 + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "names": ["L_ORDERKEY", "L_LINENUMBER", "count(1)"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_aggregate_generic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_group_by_cube-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_group_by_cube-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_group_by_cube-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_group_by_cube-spark_plan.json b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_group_by_cube-spark_plan.json new file mode 100644 index 00000000..958c75f7 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_group_by_cube-spark_plan.json @@ -0,0 +1,399 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_aggregate_generic.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "count:any", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5, 6] + }, + "hint": { + "outputNames": ["L_ORDERKEY", "L_LINENUMBER", "count(1)"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "expand": { + "common": { + "direct": { + }, + "hint": { + "outputNames": ["L_ORDERKEY", "L_LINENUMBER", "spark_grouping_id"] + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17] + }, + "hint": { + "outputNames": ["L_ORDERKEY", "L_LINENUMBER"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem_small.parquet", + "length": "2758", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "fields": [{ + "switchingField": { + "duplicates": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "literal": { + "null": { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + } + } + }, { + "literal": { + "null": { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + } + } + }] + } + }, { + "switchingField": { + "duplicates": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "literal": { + "null": { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "literal": { + "null": { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + } + } + }] + } + }, { + "switchingField": { + "duplicates": [{ + "literal": { + "i64": "0" + } + }, { + "literal": { + "i64": "1" + } + }, { + "literal": { + "i64": "2" + } + }, { + "literal": { + "i64": "3" + } + }] + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 1, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "i64": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "literal": { + "i32": 1 + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "names": ["L_ORDERKEY", "L_LINENUMBER", "count(1)"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_aggregate_generic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_group_by_rollup-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_group_by_rollup-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_group_by_rollup-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_group_by_rollup-spark_plan.json b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_group_by_rollup-spark_plan.json new file mode 100644 index 00000000..1dc609d0 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_group_by_rollup-spark_plan.json @@ -0,0 +1,377 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_aggregate_generic.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "count:any", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5, 6] + }, + "hint": { + "outputNames": ["L_ORDERKEY", "L_LINENUMBER", "count(1)"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "expand": { + "common": { + "direct": { + }, + "hint": { + "outputNames": ["L_ORDERKEY", "L_LINENUMBER", "spark_grouping_id"] + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17] + }, + "hint": { + "outputNames": ["L_ORDERKEY", "L_LINENUMBER"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem_small.parquet", + "length": "2758", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "fields": [{ + "switchingField": { + "duplicates": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "literal": { + "null": { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + } + } + }] + } + }, { + "switchingField": { + "duplicates": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "literal": { + "null": { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + } + } + }, { + "literal": { + "null": { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + } + } + }] + } + }, { + "switchingField": { + "duplicates": [{ + "literal": { + "i64": "0" + } + }, { + "literal": { + "i64": "1" + } + }, { + "literal": { + "i64": "3" + } + }] + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 1, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "i64": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "literal": { + "i32": 1 + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "names": ["L_ORDERKEY", "L_LINENUMBER", "count(1)"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_aggregate_generic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_grouping_set-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_grouping_set-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_grouping_set-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_grouping_set-spark_plan.json b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_grouping_set-spark_plan.json new file mode 100644 index 00000000..059bb86b --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/aggregate/aggregate_with_grouping_set-spark_plan.json @@ -0,0 +1,385 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_arithmetic_decimal.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "sum:dec", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5, 6] + }, + "hint": { + "outputNames": ["sum(L_EXTENDEDPRICE)", "L_LINENUMBER", "L_ORDERKEY"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "expand": { + "common": { + "direct": { + }, + "hint": { + "outputNames": ["l_extendedprice", "L_LINENUMBER", "L_ORDERKEY", "spark_grouping_id"] + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17, 18] + }, + "hint": { + "outputNames": ["l_extendedprice", "L_LINENUMBER", "L_ORDERKEY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem_small.parquet", + "length": "2758", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "fields": [{ + "switchingField": { + "duplicates": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, { + "switchingField": { + "duplicates": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "literal": { + "null": { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + } + } + }] + } + }, { + "switchingField": { + "duplicates": [{ + "literal": { + "null": { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, { + "switchingField": { + "duplicates": [{ + "literal": { + "i64": "1" + } + }, { + "literal": { + "i64": "2" + } + }] + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 1, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 2, + "precision": 25, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "names": ["sum(L_EXTENDEDPRICE)", "L_LINENUMBER", "L_ORDERKEY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/aggregate/computation_between_aggregates-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/aggregate/computation_between_aggregates-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/aggregate/computation_between_aggregates-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/aggregate/computation_between_aggregates-spark_plan.json b/substrait_consumer/snapshots/producer/relation/aggregate/computation_between_aggregates-spark_plan.json new file mode 100644 index 00000000..d7d465e0 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/aggregate/computation_between_aggregates-spark_plan.json @@ -0,0 +1,234 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_arithmetic_decimal.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "avg:dec", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 2, + "name": "max:dec", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 3, + "name": "add:dec_dec", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [2] + }, + "hint": { + "outputNames": ["(avg(O_TOTALPRICE) + max(O_TOTALPRICE))"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [9] + }, + "hint": { + "outputNames": ["o_totalprice"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders_small.parquet", + "length": "1385", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + }], + "measures": [{ + "measure": { + "functionReference": 1, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 6, + "precision": 19, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }, { + "measure": { + "functionReference": 2, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "scalarFunction": { + "functionReference": 3, + "outputType": { + "decimal": { + "scale": 6, + "precision": 20, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "names": ["(avg(O_TOTALPRICE) + max(O_TOTALPRICE))"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/aggregate/compute_within_aggregate-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/aggregate/compute_within_aggregate-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/aggregate/compute_within_aggregate-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/aggregate/compute_within_aggregate-spark_plan.json b/substrait_consumer/snapshots/producer/relation/aggregate/compute_within_aggregate-spark_plan.json new file mode 100644 index 00000000..253856d4 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/aggregate/compute_within_aggregate-spark_plan.json @@ -0,0 +1,199 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_arithmetic_decimal.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "multiply:dec_dec", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 2, + "name": "avg:dec", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [1] + }, + "hint": { + "outputNames": ["avg((O_TOTALPRICE * 10))"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [9] + }, + "hint": { + "outputNames": ["o_totalprice"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders_small.parquet", + "length": "1385", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + }], + "measures": [{ + "measure": { + "functionReference": 2, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 6, + "precision": 22, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "decimal": { + "scale": 2, + "precision": 18, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "decimal": { + "value": "CgAAAAAAAAAAAAAAAAAAAA==", + "precision": 2 + } + } + } + }] + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["avg((O_TOTALPRICE * 10))"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/aggregate/multiple_measure_aggregate-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/aggregate/multiple_measure_aggregate-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/aggregate/multiple_measure_aggregate-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/aggregate/multiple_measure_aggregate-spark_plan.json b/substrait_consumer/snapshots/producer/relation/aggregate/multiple_measure_aggregate-spark_plan.json new file mode 100644 index 00000000..730050ce --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/aggregate/multiple_measure_aggregate-spark_plan.json @@ -0,0 +1,253 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_arithmetic_decimal.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "min:dec", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 2, + "name": "max:dec", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 3, + "name": "avg:dec", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [3, 4, 5] + }, + "hint": { + "outputNames": ["min(O_TOTALPRICE)", "max(O_TOTALPRICE)", "avg(O_TOTALPRICE)"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [9] + }, + "hint": { + "outputNames": ["o_totalprice"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders_small.parquet", + "length": "1385", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + }], + "measures": [{ + "measure": { + "functionReference": 1, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }, { + "measure": { + "functionReference": 2, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }, { + "measure": { + "functionReference": 3, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 6, + "precision": 19, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["min(O_TOTALPRICE)", "max(O_TOTALPRICE)", "avg(O_TOTALPRICE)"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/aggregate/single_measure_aggregate-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/aggregate/single_measure_aggregate-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/aggregate/single_measure_aggregate-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/aggregate/single_measure_aggregate-spark_plan.json b/substrait_consumer/snapshots/producer/relation/aggregate/single_measure_aggregate-spark_plan.json new file mode 100644 index 00000000..1e6ce923 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/aggregate/single_measure_aggregate-spark_plan.json @@ -0,0 +1,201 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_aggregate_generic.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "count:any", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [1] + }, + "hint": { + "outputNames": ["count(L_PARTKEY)"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16] + }, + "hint": { + "outputNames": ["l_partkey"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem_small.parquet", + "length": "2758", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + }], + "measures": [{ + "measure": { + "functionReference": 1, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "i64": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["count(L_PARTKEY)"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_aggregate_generic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/ddl/create_view-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/ddl/create_view-spark_outcome.txt new file mode 100644 index 00000000..38f2b192 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/ddl/create_view-spark_outcome.txt @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/having-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/filter/having-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/having-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/having-spark_plan.json b/substrait_consumer/snapshots/producer/relation/filter/having-spark_plan.json new file mode 100644 index 00000000..2c267c9e --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/having-spark_plan.json @@ -0,0 +1,287 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_aggregate_generic.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "count:any", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "gt:any_any", + "extensionUrnReference": 2 + } + }], + "relations": [{ + "root": { + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [2, 3] + }, + "hint": { + "outputNames": ["L_QUANTITY", "count(1)"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16] + }, + "hint": { + "outputNames": ["l_quantity"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 1, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "i64": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "literal": { + "i32": 1 + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "condition": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "i64": "12100" + } + } + }] + } + } + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "names": ["L_QUANTITY", "count(1)"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_aggregate_generic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/where_and-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/filter/where_and-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/where_and-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/where_and-spark_plan.json b/substrait_consumer/snapshots/producer/relation/filter/where_and-spark_plan.json new file mode 100644 index 00000000..aab04669 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/where_and-spark_plan.json @@ -0,0 +1,323 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 3, + "name": "equal:any_any", + "extensionUrnReference": 2 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17] + }, + "hint": { + "outputNames": ["L_ORDERKEY", "L_SHIPINSTRUCT"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 13 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "i64": "2" + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 13 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "TAKE BACK RETURN" + } + } + }] + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 13 + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["L_ORDERKEY", "L_SHIPINSTRUCT"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/where_between-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/filter/where_between-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/where_between-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/where_between-spark_plan.json b/substrait_consumer/snapshots/producer/relation/filter/where_between-spark_plan.json new file mode 100644 index 00000000..ee98c671 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/where_between-spark_plan.json @@ -0,0 +1,293 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 3, + "name": "gte:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 4, + "name": "lte:any_any", + "extensionUrnReference": 2 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16] + }, + "hint": { + "outputNames": ["L_ORDERKEY"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "i64": "20" + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "i64": "50" + } + } + }] + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "offset": "0", + "count": "20" + } + }, + "names": ["L_ORDERKEY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/where_equal_multi_col-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/filter/where_equal_multi_col-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/where_equal_multi_col-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/where_equal_multi_col-spark_plan.json b/substrait_consumer/snapshots/producer/relation/filter/where_equal_multi_col-spark_plan.json new file mode 100644 index 00000000..99719626 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/where_equal_multi_col-spark_plan.json @@ -0,0 +1,321 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 3, + "name": "equal:any_any", + "extensionUrnReference": 2 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17] + }, + "hint": { + "outputNames": ["L_DISCOUNT", "L_TAX"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "offset": "0", + "count": "20" + } + }, + "names": ["L_DISCOUNT", "L_TAX"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/where_gt_multi_col-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/filter/where_gt_multi_col-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/where_gt_multi_col-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/where_gt_multi_col-spark_plan.json b/substrait_consumer/snapshots/producer/relation/filter/where_gt_multi_col-spark_plan.json new file mode 100644 index 00000000..2bc1bdc9 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/where_gt_multi_col-spark_plan.json @@ -0,0 +1,334 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 3, + "name": "gt:any_any", + "extensionUrnReference": 2 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17] + }, + "hint": { + "outputNames": ["L_DISCOUNT", "L_TAX"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "offset": "0", + "count": "20" + } + }, + "names": ["L_DISCOUNT", "L_TAX"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/where_gte_multi_col-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/filter/where_gte_multi_col-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/where_gte_multi_col-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/where_gte_multi_col-spark_plan.json b/substrait_consumer/snapshots/producer/relation/filter/where_gte_multi_col-spark_plan.json new file mode 100644 index 00000000..b7425889 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/where_gte_multi_col-spark_plan.json @@ -0,0 +1,334 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 3, + "name": "gte:any_any", + "extensionUrnReference": 2 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17] + }, + "hint": { + "outputNames": ["L_DISCOUNT", "L_TAX"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "offset": "0", + "count": "20" + } + }, + "names": ["L_DISCOUNT", "L_TAX"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/where_in-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/filter/where_in-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/where_in-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/where_in-spark_plan.json b/substrait_consumer/snapshots/producer/relation/filter/where_in-spark_plan.json new file mode 100644 index 00000000..867923b5 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/where_in-spark_plan.json @@ -0,0 +1,163 @@ +{ + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16] + }, + "hint": { + "outputNames": ["L_ORDERKEY"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "singularOrList": { + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "options": [{ + "literal": { + "i64": "1" + } + }, { + "literal": { + "i64": "2" + } + }, { + "literal": { + "i64": "3" + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["L_ORDERKEY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + } +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/where_like-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/filter/where_like-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/where_like-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/where_like-spark_plan.json b/substrait_consumer/snapshots/producer/relation/filter/where_like-spark_plan.json new file mode 100644 index 00000000..14bd6245 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/where_like-spark_plan.json @@ -0,0 +1,285 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 3, + "uri": "/functions_string.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 3, + "functionAnchor": 3, + "name": "contains:str_str", + "extensionUrnReference": 3 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17] + }, + "hint": { + "outputNames": ["L_SHIPINSTRUCT", "L_ORDERKEY"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 13 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 13 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "string": "DELIVER IN PERSON" + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 13 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "offset": "0", + "count": "20" + } + }, + "names": ["L_SHIPINSTRUCT", "L_ORDERKEY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }, { + "extensionUrnAnchor": 3, + "urn": "extension:io.substrait:functions_string" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/where_lt_multi_col-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/filter/where_lt_multi_col-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/where_lt_multi_col-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/where_lt_multi_col-spark_plan.json b/substrait_consumer/snapshots/producer/relation/filter/where_lt_multi_col-spark_plan.json new file mode 100644 index 00000000..322fced1 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/where_lt_multi_col-spark_plan.json @@ -0,0 +1,334 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 3, + "name": "lt:any_any", + "extensionUrnReference": 2 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17] + }, + "hint": { + "outputNames": ["L_DISCOUNT", "L_TAX"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "offset": "0", + "count": "20" + } + }, + "names": ["L_DISCOUNT", "L_TAX"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/where_lte_multi_col-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/filter/where_lte_multi_col-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/where_lte_multi_col-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/where_lte_multi_col-spark_plan.json b/substrait_consumer/snapshots/producer/relation/filter/where_lte_multi_col-spark_plan.json new file mode 100644 index 00000000..75eef604 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/where_lte_multi_col-spark_plan.json @@ -0,0 +1,334 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 3, + "name": "lte:any_any", + "extensionUrnReference": 2 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17] + }, + "hint": { + "outputNames": ["L_DISCOUNT", "L_TAX"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "offset": "0", + "count": "20" + } + }, + "names": ["L_DISCOUNT", "L_TAX"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/where_not_equal_multi_col-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/filter/where_not_equal_multi_col-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/where_not_equal_multi_col-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/where_not_equal_multi_col-spark_plan.json b/substrait_consumer/snapshots/producer/relation/filter/where_not_equal_multi_col-spark_plan.json new file mode 100644 index 00000000..da7e2b4f --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/where_not_equal_multi_col-spark_plan.json @@ -0,0 +1,353 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 3, + "name": "not:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 4, + "name": "equal:any_any", + "extensionUrnReference": 2 + } + }], + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17] + }, + "hint": { + "outputNames": ["L_DISCOUNT", "L_TAX"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "offset": "0", + "count": "20" + } + }, + "names": ["L_DISCOUNT", "L_TAX"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/where_or-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/filter/where_or-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/where_or-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/filter/where_or-spark_plan.json b/substrait_consumer/snapshots/producer/relation/filter/where_or-spark_plan.json new file mode 100644 index 00000000..e4dbb170 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/filter/where_or-spark_plan.json @@ -0,0 +1,244 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "or:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "equal:any_any", + "extensionUrnReference": 2 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17] + }, + "hint": { + "outputNames": ["L_ORDERKEY", "L_SHIPINSTRUCT"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem.parquet", + "length": "26368539", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "i64": "2" + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "i64": "3" + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 13 + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["L_ORDERKEY", "L_SHIPINSTRUCT"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/join/cross_join-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/join/cross_join-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/join/cross_join-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/join/cross_join-spark_plan.json b/substrait_consumer/snapshots/producer/relation/join/cross_join-spark_plan.json new file mode 100644 index 00000000..5e340626 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/join/cross_join-spark_plan.json @@ -0,0 +1,233 @@ +{ + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [3, 4, 5] + }, + "hint": { + "outputNames": ["C_CUSTKEY", "C_NAME", "O_ORDERKEY"] + } + }, + "input": { + "cross": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [8, 9] + }, + "hint": { + "outputNames": ["c_custkey", "c_name"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer_small.parquet", + "length": "1533", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [9] + }, + "hint": { + "outputNames": ["o_orderkey"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders_small.parquet", + "length": "1385", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["C_CUSTKEY", "C_NAME", "O_ORDERKEY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + } +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/join/full_join-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/join/full_join-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/join/full_join-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/join/full_join-spark_plan.json b/substrait_consumer/snapshots/producer/relation/join/full_join-spark_plan.json new file mode 100644 index 00000000..2f593eef --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/join/full_join-spark_plan.json @@ -0,0 +1,294 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "equal:any_any", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5, 6] + }, + "hint": { + "outputNames": ["C_CUSTKEY", "C_NAME", "O_ORDERKEY"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [8, 9] + }, + "hint": { + "outputNames": ["c_custkey", "c_name"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer_small.parquet", + "length": "1533", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [9, 10] + }, + "hint": { + "outputNames": ["o_orderkey", "o_custkey"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders_small.parquet", + "length": "1385", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_OUTER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["C_CUSTKEY", "C_NAME", "O_ORDERKEY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_comparison" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/join/inner_join-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/join/inner_join-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/join/inner_join-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/join/inner_join-spark_plan.json b/substrait_consumer/snapshots/producer/relation/join/inner_join-spark_plan.json new file mode 100644 index 00000000..a424db41 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/join/inner_join-spark_plan.json @@ -0,0 +1,362 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "is_not_null:any", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 2, + "name": "equal:any_any", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5, 6] + }, + "hint": { + "outputNames": ["C_CUSTKEY", "C_NAME", "O_ORDERKEY"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [8, 9] + }, + "hint": { + "outputNames": ["c_custkey", "c_name"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer_small.parquet", + "length": "1533", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [9, 10] + }, + "hint": { + "outputNames": ["o_orderkey", "o_custkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders_small.parquet", + "length": "1385", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_INNER" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["C_CUSTKEY", "C_NAME", "O_ORDERKEY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_comparison" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/join/left_anti_join-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/join/left_anti_join-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/join/left_anti_join-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/join/left_anti_join-spark_plan.json b/substrait_consumer/snapshots/producer/relation/join/left_anti_join-spark_plan.json new file mode 100644 index 00000000..e5f497d1 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/join/left_anti_join-spark_plan.json @@ -0,0 +1,275 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "equal:any_any", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [2, 3] + }, + "hint": { + "outputNames": ["C_CUSTKEY", "C_NAME"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [8, 9] + }, + "hint": { + "outputNames": ["c_custkey", "c_name"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer_small.parquet", + "length": "1533", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [9] + }, + "hint": { + "outputNames": ["O_CUSTKEY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders_small.parquet", + "length": "1385", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_LEFT_ANTI" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["C_CUSTKEY", "C_NAME"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_comparison" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/join/left_join-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/join/left_join-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/join/left_join-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/join/left_join-spark_plan.json b/substrait_consumer/snapshots/producer/relation/join/left_join-spark_plan.json new file mode 100644 index 00000000..4b681088 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/join/left_join-spark_plan.json @@ -0,0 +1,332 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "is_not_null:any", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 2, + "name": "equal:any_any", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5, 6] + }, + "hint": { + "outputNames": ["C_CUSTKEY", "C_NAME", "O_ORDERKEY"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [8, 9] + }, + "hint": { + "outputNames": ["c_custkey", "c_name"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer_small.parquet", + "length": "1533", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [9, 10] + }, + "hint": { + "outputNames": ["o_orderkey", "o_custkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders_small.parquet", + "length": "1385", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_LEFT" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["C_CUSTKEY", "C_NAME", "O_ORDERKEY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_comparison" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/join/left_semi_join-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/join/left_semi_join-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/join/left_semi_join-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/join/left_semi_join-spark_plan.json b/substrait_consumer/snapshots/producer/relation/join/left_semi_join-spark_plan.json new file mode 100644 index 00000000..8cc0f5ca --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/join/left_semi_join-spark_plan.json @@ -0,0 +1,275 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "equal:any_any", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [2, 3] + }, + "hint": { + "outputNames": ["C_CUSTKEY", "C_NAME"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [8, 9] + }, + "hint": { + "outputNames": ["c_custkey", "c_name"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer_small.parquet", + "length": "1533", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [9] + }, + "hint": { + "outputNames": ["O_CUSTKEY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders_small.parquet", + "length": "1385", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_LEFT_SEMI" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["C_CUSTKEY", "C_NAME"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_comparison" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/join/left_single_join-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/join/left_single_join-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/join/left_single_join-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/join/left_single_join-spark_plan.json b/substrait_consumer/snapshots/producer/relation/join/left_single_join-spark_plan.json new file mode 100644 index 00000000..3df9101e --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/join/left_single_join-spark_plan.json @@ -0,0 +1,493 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 3, + "name": "equal:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 4, + "name": "not:bool", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [6, 7, 8, 9, 10, 11] + }, + "hint": { + "outputNames": ["c1key", "c1name", "c1nationakey", "c2key", "c2name", "c2nationakey"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [8, 9, 10] + }, + "hint": { + "outputNames": ["c_custkey", "c_name", "c_nationkey"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer_small.parquet", + "length": "1533", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [8, 9, 10] + }, + "hint": { + "outputNames": ["c_custkey", "c_name", "c_nationkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer_small.parquet", + "length": "1533", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }] + } + }, + "type": "JOIN_TYPE_LEFT" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["c1key", "c1name", "c1nationakey", "c2key", "c2name", "c2nationakey"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/join/right_anti_join-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/join/right_anti_join-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/join/right_anti_join-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/join/right_anti_join-spark_plan.json b/substrait_consumer/snapshots/producer/relation/join/right_anti_join-spark_plan.json new file mode 100644 index 00000000..2a626083 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/join/right_anti_join-spark_plan.json @@ -0,0 +1,312 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "equal:any_any", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [2, 3] + }, + "hint": { + "outputNames": ["O_ORDERKEY", "O_CUSTKEY"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [9, 10] + }, + "hint": { + "outputNames": ["o_orderkey", "o_custkey"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders_small.parquet", + "length": "1385", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [16] + }, + "hint": { + "outputNames": ["L_ORDERKEY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem_small.parquet", + "length": "2758", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_LEFT_ANTI" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["O_ORDERKEY", "O_CUSTKEY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_comparison" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/join/right_join-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/join/right_join-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/join/right_join-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/join/right_join-spark_plan.json b/substrait_consumer/snapshots/producer/relation/join/right_join-spark_plan.json new file mode 100644 index 00000000..b2472667 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/join/right_join-spark_plan.json @@ -0,0 +1,331 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "is_not_null:any", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 2, + "name": "equal:any_any", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [4, 5, 6] + }, + "hint": { + "outputNames": ["C_CUSTKEY", "C_NAME", "O_ORDERKEY"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [8, 9] + }, + "hint": { + "outputNames": ["c_custkey", "c_name"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer_small.parquet", + "length": "1533", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [9, 10] + }, + "hint": { + "outputNames": ["o_orderkey", "o_custkey"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders_small.parquet", + "length": "1385", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_RIGHT" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["C_CUSTKEY", "C_NAME", "O_ORDERKEY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_comparison" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/join/right_semi_join-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/join/right_semi_join-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/join/right_semi_join-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/join/right_semi_join-spark_plan.json b/substrait_consumer/snapshots/producer/relation/join/right_semi_join-spark_plan.json new file mode 100644 index 00000000..e4471597 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/join/right_semi_join-spark_plan.json @@ -0,0 +1,275 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "equal:any_any", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [2, 3] + }, + "hint": { + "outputNames": ["O_ORDERKEY", "O_CUSTKEY"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [9, 10] + }, + "hint": { + "outputNames": ["o_orderkey", "o_custkey"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders_small.parquet", + "length": "1385", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [8] + }, + "hint": { + "outputNames": ["C_CUSTKEY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer_small.parquet", + "length": "1533", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_LEFT_SEMI" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["O_ORDERKEY", "O_CUSTKEY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_comparison" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/join/right_single_join-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/join/right_single_join-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/join/right_single_join-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/join/right_single_join-spark_plan.json b/substrait_consumer/snapshots/producer/relation/join/right_single_join-spark_plan.json new file mode 100644 index 00000000..8384c8d6 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/join/right_single_join-spark_plan.json @@ -0,0 +1,493 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_boolean.yaml" + }, { + "extensionUriAnchor": 2, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "and:bool", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "is_not_null:any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 3, + "name": "equal:any_any", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 4, + "name": "not:bool", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [6, 7, 8, 9, 10, 11] + }, + "hint": { + "outputNames": ["c1key", "c1name", "c1nationakey", "c2key", "c2name", "c2nationakey"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [8, 9, 10] + }, + "hint": { + "outputNames": ["c_custkey", "c_name", "c_nationkey"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer_small.parquet", + "length": "1533", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [8, 9, 10] + }, + "hint": { + "outputNames": ["c_custkey", "c_name", "c_nationkey"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer_small.parquet", + "length": "1533", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 4, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + } + }] + } + } + }] + } + } + }] + } + }, + "type": "JOIN_TYPE_RIGHT" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["c1key", "c1name", "c1nationakey", "c2key", "c2name", "c2nationakey"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_comparison" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_boolean" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/project/count_distinct_in_project-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/project/count_distinct_in_project-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/project/count_distinct_in_project-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/project/count_distinct_in_project-spark_plan.json b/substrait_consumer/snapshots/producer/relation/project/count_distinct_in_project-spark_plan.json new file mode 100644 index 00000000..5b8d5bda --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/project/count_distinct_in_project-spark_plan.json @@ -0,0 +1,201 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_aggregate_generic.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "count:any", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [1] + }, + "hint": { + "outputNames": ["count(DISTINCT L_EXTENDEDPRICE)"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16] + }, + "hint": { + "outputNames": ["l_extendedprice"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem_small.parquet", + "length": "2758", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + }], + "measures": [{ + "measure": { + "functionReference": 1, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "i64": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "invocation": "AGGREGATION_INVOCATION_DISTINCT", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["count(DISTINCT L_EXTENDEDPRICE)"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_aggregate_generic" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/project/distinct_in_project-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/project/distinct_in_project-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/project/distinct_in_project-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/project/distinct_in_project-spark_plan.json b/substrait_consumer/snapshots/producer/relation/project/distinct_in_project-spark_plan.json new file mode 100644 index 00000000..dd7adab8 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/project/distinct_in_project-spark_plan.json @@ -0,0 +1,171 @@ +{ + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [1] + }, + "hint": { + "outputNames": ["L_LINESTATUS"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16] + }, + "hint": { + "outputNames": ["L_LINESTATUS"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem_small.parquet", + "length": "2758", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 9 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["L_LINESTATUS"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + } +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/project/extended_project-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/project/extended_project-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/project/extended_project-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/project/extended_project-spark_plan.json b/substrait_consumer/snapshots/producer/relation/project/extended_project-spark_plan.json new file mode 100644 index 00000000..46424ae5 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/project/extended_project-spark_plan.json @@ -0,0 +1,178 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_arithmetic_decimal.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "multiply:dec_dec", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17] + }, + "hint": { + "outputNames": ["L_QUANTITY", "MULTI_PRICE"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem_small.parquet", + "length": "2758", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 4 + } + }, + "rootReference": { + } + } + }, { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "decimal": { + "scale": 2, + "precision": 18, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "literal": { + "decimal": { + "value": "CgAAAAAAAAAAAAAAAAAAAA==", + "precision": 2 + } + } + } + }] + } + }] + } + }, + "names": ["L_QUANTITY", "MULTI_PRICE"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/project/project_all_col-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/project/project_all_col-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/project/project_all_col-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/project/project_all_col-spark_plan.json b/substrait_consumer/snapshots/producer/relation/project/project_all_col-spark_plan.json new file mode 100644 index 00000000..bf5e1250 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/project/project_all_col-spark_plan.json @@ -0,0 +1,46 @@ +{ + "relations": [{ + "root": { + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["r_regionkey", "r_name", "r_comment"], + "struct": { + "types": [{ + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/region_small.parquet", + "length": "1058", + "parquet": { + } + }] + } + } + }, + "names": ["r_regionkey", "r_name", "r_comment"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + } +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/project/project_multi_col-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/project/project_multi_col-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/project/project_multi_col-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/project/project_multi_col-spark_plan.json b/substrait_consumer/snapshots/producer/relation/project/project_multi_col-spark_plan.json new file mode 100644 index 00000000..3b755a5e --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/project/project_multi_col-spark_plan.json @@ -0,0 +1,139 @@ +{ + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [16, 17] + }, + "hint": { + "outputNames": ["L_DISCOUNT", "L_TAX"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem_small.parquet", + "length": "2758", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 6 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 7 + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["L_DISCOUNT", "L_TAX"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + } +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/project/project_single_col-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/project/project_single_col-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/project/project_single_col-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/project/project_single_col-spark_plan.json b/substrait_consumer/snapshots/producer/relation/project/project_single_col-spark_plan.json new file mode 100644 index 00000000..4071bdd3 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/project/project_single_col-spark_plan.json @@ -0,0 +1,106 @@ +{ + "relations": [{ + "root": { + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/lineitem_small.parquet", + "length": "2758", + "parquet": { + } + }] + } + } + }, + "names": ["l_orderkey", "l_partkey", "l_suppkey", "l_linenumber", "l_quantity", "l_extendedprice", "l_discount", "l_tax", "l_returnflag", "l_linestatus", "l_shipdate", "l_commitdate", "l_receiptdate", "l_shipinstruct", "l_shipmode", "l_comment"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + } +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/project/subquery_in_project-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/project/subquery_in_project-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/project/subquery_in_project-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/project/subquery_in_project-spark_plan.json b/substrait_consumer/snapshots/producer/relation/project/subquery_in_project-spark_plan.json new file mode 100644 index 00000000..2ff3e99b --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/project/subquery_in_project-spark_plan.json @@ -0,0 +1,405 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 2, + "uri": "/functions_arithmetic_decimal.yaml" + }, { + "extensionUriAnchor": 1, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "is_not_null:any", + "extensionUrnReference": 1 + } + }, { + "extensionFunction": { + "extensionUriReference": 2, + "functionAnchor": 2, + "name": "sum:dec", + "extensionUrnReference": 2 + } + }, { + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 3, + "name": "equal:any_any", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [3, 4] + }, + "hint": { + "outputNames": ["C_CUSTKEY", "total_price"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [8] + }, + "hint": { + "outputNames": ["c_custkey"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer_small.parquet", + "length": "1533", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [2, 3] + }, + "hint": { + "outputNames": ["sum(O_TOTALPRICE)", "O_CUSTKEY"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [9, 10] + }, + "hint": { + "outputNames": ["o_custkey", "o_totalprice"] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders_small.parquet", + "length": "1385", + "parquet": { + } + }] + } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 2, + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "outputType": { + "decimal": { + "scale": 2, + "precision": 25, + "nullability": "NULLABILITY_NULLABLE" + } + }, + "invocation": "AGGREGATION_INVOCATION_ALL", + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_LEFT" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["C_CUSTKEY", "total_price"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 2, + "urn": "extension:io.substrait:functions_arithmetic_decimal" + }, { + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_comparison" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/read/read_named_table-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/read/read_named_table-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/read/read_named_table-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/read/read_named_table-spark_plan.json b/substrait_consumer/snapshots/producer/relation/read/read_named_table-spark_plan.json new file mode 100644 index 00000000..98b37c41 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/read/read_named_table-spark_plan.json @@ -0,0 +1,78 @@ +{ + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5] + }, + "hint": { + "outputNames": ["PS_PARTKEY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp_small.parquet", + "length": "1739", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["PS_PARTKEY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + } +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/set/except-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/set/except-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/set/except-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/set/except-spark_plan.json b/substrait_consumer/snapshots/producer/relation/set/except-spark_plan.json new file mode 100644 index 00000000..c1cd1d96 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/set/except-spark_plan.json @@ -0,0 +1,276 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "is_not_distinct_from:any_any", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [1] + }, + "hint": { + "outputNames": ["o_totalprice"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [9] + }, + "hint": { + "outputNames": ["o_totalprice"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["o_orderkey", "o_custkey", "o_orderstatus", "o_totalprice", "o_orderdate", "o_orderpriority", "o_clerk", "o_shippriority", "o_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "date": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/orders_small.parquet", + "length": "1385", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [8] + }, + "hint": { + "outputNames": ["c_acctbal"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer_small.parquet", + "length": "1533", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 5 + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_LEFT_ANTI" + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["o_totalprice"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_comparison" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/set/intersect-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/set/intersect-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/set/intersect-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/set/intersect-spark_plan.json b/substrait_consumer/snapshots/producer/relation/set/intersect-spark_plan.json new file mode 100644 index 00000000..40834ef3 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/set/intersect-spark_plan.json @@ -0,0 +1,275 @@ +{ + "extensionUris": [{ + "extensionUriAnchor": 1, + "uri": "/functions_comparison.yaml" + }], + "extensions": [{ + "extensionFunction": { + "extensionUriReference": 1, + "functionAnchor": 1, + "name": "is_not_distinct_from:any_any", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [1] + }, + "hint": { + "outputNames": ["C_NATIONKEY"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [1] + }, + "hint": { + "outputNames": ["C_NATIONKEY"] + } + }, + "input": { + "join": { + "common": { + "direct": { + } + }, + "left": { + "project": { + "common": { + "emit": { + "outputMapping": [8] + }, + "hint": { + "outputNames": ["c_nationkey"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer_small.parquet", + "length": "1533", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "right": { + "project": { + "common": { + "emit": { + "outputMapping": [4] + }, + "hint": { + "outputNames": ["N_NATIONKEY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["n_nationkey", "n_name", "n_regionkey", "n_comment"], + "struct": { + "types": [{ + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/nation_small.parquet", + "length": "2248", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "expression": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + } + }, { + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + } + }] + } + }, + "type": "JOIN_TYPE_LEFT_SEMI" + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "names": ["C_NATIONKEY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + }, + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_comparison" + }] +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/set/union_all-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/set/union_all-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/set/union_all-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/set/union_all-spark_plan.json b/substrait_consumer/snapshots/producer/relation/set/union_all-spark_plan.json new file mode 100644 index 00000000..7e50a9a2 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/set/union_all-spark_plan.json @@ -0,0 +1,160 @@ +{ + "relations": [{ + "root": { + "input": { + "set": { + "common": { + "direct": { + } + }, + "inputs": [{ + "project": { + "common": { + "emit": { + "outputMapping": [8] + }, + "hint": { + "outputNames": ["C_NATIONKEY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer_small.parquet", + "length": "1533", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, { + "project": { + "common": { + "emit": { + "outputMapping": [4] + }, + "hint": { + "outputNames": ["N_NATIONKEY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["n_nationkey", "n_name", "n_regionkey", "n_comment"], + "struct": { + "types": [{ + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/nation_small.parquet", + "length": "2248", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }], + "op": "SET_OP_UNION_ALL" + } + }, + "names": ["C_NATIONKEY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + } +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/set/union_distinct-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/set/union_distinct-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/set/union_distinct-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/set/union_distinct-spark_plan.json b/substrait_consumer/snapshots/producer/relation/set/union_distinct-spark_plan.json new file mode 100644 index 00000000..0e35e939 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/set/union_distinct-spark_plan.json @@ -0,0 +1,223 @@ +{ + "relations": [{ + "root": { + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [1] + }, + "hint": { + "outputNames": ["C_NATIONKEY"] + } + }, + "input": { + "aggregate": { + "common": { + "direct": { + } + }, + "input": { + "set": { + "common": { + "direct": { + } + }, + "inputs": [{ + "project": { + "common": { + "emit": { + "outputMapping": [8] + }, + "hint": { + "outputNames": ["C_NATIONKEY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["c_custkey", "c_name", "c_address", "c_nationkey", "c_phone", "c_acctbal", "c_mktsegment", "c_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/customer_small.parquet", + "length": "1533", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, { + "project": { + "common": { + "emit": { + "outputMapping": [4] + }, + "hint": { + "outputNames": ["N_NATIONKEY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["n_nationkey", "n_name", "n_regionkey", "n_comment"], + "struct": { + "types": [{ + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i32": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/nation_small.parquet", + "length": "2248", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }], + "op": "SET_OP_UNION_ALL" + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + }] + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "names": ["C_NATIONKEY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + } +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/sort/multi_col_asc-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/sort/multi_col_asc-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/sort/multi_col_asc-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/sort/multi_col_asc-spark_plan.json b/substrait_consumer/snapshots/producer/relation/sort/multi_col_asc-spark_plan.json new file mode 100644 index 00000000..1a4b52cf --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/sort/multi_col_asc-spark_plan.json @@ -0,0 +1,133 @@ +{ + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6] + }, + "hint": { + "outputNames": ["PS_SUPPLYCOST", "PS_AVAILQTY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["PS_SUPPLYCOST", "PS_AVAILQTY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + } +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/sort/multi_col_asc_desc-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/sort/multi_col_asc_desc-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/sort/multi_col_asc_desc-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/sort/multi_col_asc_desc-spark_plan.json b/substrait_consumer/snapshots/producer/relation/sort/multi_col_asc_desc-spark_plan.json new file mode 100644 index 00000000..364d384f --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/sort/multi_col_asc_desc-spark_plan.json @@ -0,0 +1,133 @@ +{ + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6] + }, + "hint": { + "outputNames": ["PS_SUPPLYCOST", "PS_AVAILQTY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_DESC_NULLS_LAST" + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["PS_SUPPLYCOST", "PS_AVAILQTY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + } +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/sort/multi_col_desc-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/sort/multi_col_desc-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/sort/multi_col_desc-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/sort/multi_col_desc-spark_plan.json b/substrait_consumer/snapshots/producer/relation/sort/multi_col_desc-spark_plan.json new file mode 100644 index 00000000..82e84b3b --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/sort/multi_col_desc-spark_plan.json @@ -0,0 +1,110 @@ +{ + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5] + }, + "hint": { + "outputNames": ["PS_SUPPLYCOST"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_DESC_NULLS_LAST" + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["PS_SUPPLYCOST"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + } +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/sort/multi_col_desc_asc-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/sort/multi_col_desc_asc-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/sort/multi_col_desc_asc-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/sort/multi_col_desc_asc-spark_plan.json b/substrait_consumer/snapshots/producer/relation/sort/multi_col_desc_asc-spark_plan.json new file mode 100644 index 00000000..db875668 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/sort/multi_col_desc_asc-spark_plan.json @@ -0,0 +1,133 @@ +{ + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6] + }, + "hint": { + "outputNames": ["PS_SUPPLYCOST", "PS_AVAILQTY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_DESC_NULLS_LAST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["PS_SUPPLYCOST", "PS_AVAILQTY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + } +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/sort/order_by_col_number-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/sort/order_by_col_number-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/sort/order_by_col_number-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/sort/order_by_col_number-spark_plan.json b/substrait_consumer/snapshots/producer/relation/sort/order_by_col_number-spark_plan.json new file mode 100644 index 00000000..1a4b52cf --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/sort/order_by_col_number-spark_plan.json @@ -0,0 +1,133 @@ +{ + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5, 6] + }, + "hint": { + "outputNames": ["PS_SUPPLYCOST", "PS_AVAILQTY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }, { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }, { + "expr": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["PS_SUPPLYCOST", "PS_AVAILQTY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + } +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/sort/single_col_asc-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/sort/single_col_asc-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/sort/single_col_asc-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/sort/single_col_asc-spark_plan.json b/substrait_consumer/snapshots/producer/relation/sort/single_col_asc-spark_plan.json new file mode 100644 index 00000000..555830f9 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/sort/single_col_asc-spark_plan.json @@ -0,0 +1,110 @@ +{ + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5] + }, + "hint": { + "outputNames": ["PS_SUPPLYCOST"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["PS_SUPPLYCOST"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + } +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/sort/single_col_default_sort-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/sort/single_col_default_sort-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/sort/single_col_default_sort-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/sort/single_col_default_sort-spark_plan.json b/substrait_consumer/snapshots/producer/relation/sort/single_col_default_sort-spark_plan.json new file mode 100644 index 00000000..e881ff7f --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/sort/single_col_default_sort-spark_plan.json @@ -0,0 +1,110 @@ +{ + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5] + }, + "hint": { + "outputNames": ["PS_AVAILQTY"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 2 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_ASC_NULLS_FIRST" + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["PS_AVAILQTY"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + } +} \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/sort/single_col_desc-spark_outcome.txt b/substrait_consumer/snapshots/producer/relation/sort/single_col_desc-spark_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/sort/single_col_desc-spark_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/snapshots/producer/relation/sort/single_col_desc-spark_plan.json b/substrait_consumer/snapshots/producer/relation/sort/single_col_desc-spark_plan.json new file mode 100644 index 00000000..82e84b3b --- /dev/null +++ b/substrait_consumer/snapshots/producer/relation/sort/single_col_desc-spark_plan.json @@ -0,0 +1,110 @@ +{ + "relations": [{ + "root": { + "input": { + "fetch": { + "common": { + "direct": { + } + }, + "input": { + "sort": { + "common": { + "direct": { + } + }, + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [5] + }, + "hint": { + "outputNames": ["PS_SUPPLYCOST"] + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "decimal": { + "scale": 2, + "precision": 15, + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + }, + "localFiles": { + "items": [{ + "uriFile": "file:///tmp/substrait-io/consumer-testing/substrait_consumer/data/tpch_parquet/partsupp.parquet", + "length": "4367655", + "parquet": { + } + }] + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 3 + } + }, + "rootReference": { + } + } + }] + } + }, + "sorts": [{ + "expr": { + "selection": { + "directReference": { + "structField": { + } + }, + "rootReference": { + } + } + }, + "direction": "SORT_DIRECTION_DESC_NULLS_LAST" + }] + } + }, + "offset": "0", + "count": "10" + } + }, + "names": ["PS_SUPPLYCOST"] + } + }], + "version": { + "minorNumber": 77, + "producer": "substrait-spark" + } +} \ No newline at end of file diff --git a/substrait_consumer/testdata/function/arithmetic/aggregate/avg.json b/substrait_consumer/testdata/function/arithmetic/aggregate/avg.json index 99d42068..3e770749 100644 --- a/substrait_consumer/testdata/function/arithmetic/aggregate/avg.json +++ b/substrait_consumer/testdata/function/arithmetic/aggregate/avg.json @@ -6,7 +6,8 @@ "sql_query": { "producers": [ "datafusion", - "duckdb" + "duckdb", + "spark" ], "query": "SELECT round(avg(PS_SUPPLYCOST), 2) AS AVG_SUPPLYCOST FROM '{partsupp}';" }, diff --git a/substrait_consumer/testdata/function/arithmetic/aggregate/count.json b/substrait_consumer/testdata/function/arithmetic/aggregate/count.json index fbe6f37c..13effa9a 100644 --- a/substrait_consumer/testdata/function/arithmetic/aggregate/count.json +++ b/substrait_consumer/testdata/function/arithmetic/aggregate/count.json @@ -7,7 +7,8 @@ "producers": [ "datafusion", "duckdb", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT count(PS_SUPPLYCOST) AS COUNT_SUPPLYCOST FROM '{partsupp}';" }, diff --git a/substrait_consumer/testdata/function/arithmetic/aggregate/count_star.json b/substrait_consumer/testdata/function/arithmetic/aggregate/count_star.json index ea1a236f..829f01d9 100644 --- a/substrait_consumer/testdata/function/arithmetic/aggregate/count_star.json +++ b/substrait_consumer/testdata/function/arithmetic/aggregate/count_star.json @@ -7,7 +7,8 @@ "producers": [ "datafusion", "duckdb", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT count(*) FROM '{partsupp}';" }, diff --git a/substrait_consumer/testdata/function/arithmetic/aggregate/max.json b/substrait_consumer/testdata/function/arithmetic/aggregate/max.json index d9b23c65..f59ead4b 100644 --- a/substrait_consumer/testdata/function/arithmetic/aggregate/max.json +++ b/substrait_consumer/testdata/function/arithmetic/aggregate/max.json @@ -7,7 +7,8 @@ "producers": [ "datafusion", "duckdb", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT max(PS_SUPPLYCOST) AS MAX_SUPPLYCOST FROM '{partsupp}';" }, diff --git a/substrait_consumer/testdata/function/arithmetic/aggregate/min.json b/substrait_consumer/testdata/function/arithmetic/aggregate/min.json index 3d3b9ee8..5958c3a2 100644 --- a/substrait_consumer/testdata/function/arithmetic/aggregate/min.json +++ b/substrait_consumer/testdata/function/arithmetic/aggregate/min.json @@ -7,7 +7,8 @@ "producers": [ "datafusion", "duckdb", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT min(PS_SUPPLYCOST) AS MIN_SUPPLYCOST FROM '{partsupp}';" }, diff --git a/substrait_consumer/testdata/function/arithmetic/aggregate/std_dev.json b/substrait_consumer/testdata/function/arithmetic/aggregate/std_dev.json index 4077fa3d..8effd6c1 100644 --- a/substrait_consumer/testdata/function/arithmetic/aggregate/std_dev.json +++ b/substrait_consumer/testdata/function/arithmetic/aggregate/std_dev.json @@ -5,7 +5,8 @@ }, "sql_query": { "producers": [ - "duckdb" + "duckdb", + "spark" ], "query": "SELECT round(stddev(PS_SUPPLYCOST), 2) AS STDDEV_SUPPLYCOST FROM '{partsupp}';" }, diff --git a/substrait_consumer/testdata/function/arithmetic/aggregate/sum.json b/substrait_consumer/testdata/function/arithmetic/aggregate/sum.json index df98ea4e..99bdbb62 100644 --- a/substrait_consumer/testdata/function/arithmetic/aggregate/sum.json +++ b/substrait_consumer/testdata/function/arithmetic/aggregate/sum.json @@ -7,7 +7,8 @@ "producers": [ "datafusion", "duckdb", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT sum(PS_SUPPLYCOST) AS SUM_SUPPLYCOST FROM '{partsupp}';" }, diff --git a/substrait_consumer/testdata/function/arithmetic/scalar/acos.json b/substrait_consumer/testdata/function/arithmetic/scalar/acos.json index 9241925a..abc4086a 100644 --- a/substrait_consumer/testdata/function/arithmetic/scalar/acos.json +++ b/substrait_consumer/testdata/function/arithmetic/scalar/acos.json @@ -7,7 +7,8 @@ "producers": [ "datafusion", "duckdb", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT round(acos(CAST(l_tax AS DOUBLE)), 2) AS ACOS_TAX FROM '{lineitem}' LIMIT 10;" }, diff --git a/substrait_consumer/testdata/function/arithmetic/scalar/add.json b/substrait_consumer/testdata/function/arithmetic/scalar/add.json index ba5c0179..ecee290e 100644 --- a/substrait_consumer/testdata/function/arithmetic/scalar/add.json +++ b/substrait_consumer/testdata/function/arithmetic/scalar/add.json @@ -7,7 +7,8 @@ "producers": [ "datafusion", "duckdb", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT PS_PARTKEY, PS_SUPPKEY, PS_PARTKEY + PS_SUPPKEY AS ADD_KEY FROM '{partsupp}' LIMIT 10;" }, diff --git a/substrait_consumer/testdata/function/arithmetic/scalar/asin.json b/substrait_consumer/testdata/function/arithmetic/scalar/asin.json index 467ccdac..f6146868 100644 --- a/substrait_consumer/testdata/function/arithmetic/scalar/asin.json +++ b/substrait_consumer/testdata/function/arithmetic/scalar/asin.json @@ -7,7 +7,8 @@ "producers": [ "datafusion", "duckdb", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT round(asin(CAST(l_tax AS DOUBLE)), 2) AS ASIN_TAX FROM '{lineitem}' LIMIT 10;" }, diff --git a/substrait_consumer/testdata/function/arithmetic/scalar/atan.json b/substrait_consumer/testdata/function/arithmetic/scalar/atan.json index ca8caa5d..6c2ed3f9 100644 --- a/substrait_consumer/testdata/function/arithmetic/scalar/atan.json +++ b/substrait_consumer/testdata/function/arithmetic/scalar/atan.json @@ -7,7 +7,8 @@ "producers": [ "datafusion", "duckdb", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT round(atan(CAST(l_tax AS DOUBLE)), 2) AS ATAN_TAX FROM '{lineitem}' LIMIT 10;" }, diff --git a/substrait_consumer/testdata/function/arithmetic/scalar/atan2.json b/substrait_consumer/testdata/function/arithmetic/scalar/atan2.json index 95b0589a..967f66d1 100644 --- a/substrait_consumer/testdata/function/arithmetic/scalar/atan2.json +++ b/substrait_consumer/testdata/function/arithmetic/scalar/atan2.json @@ -7,7 +7,8 @@ "producers": [ "datafusion", "duckdb", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT round(atan2(CAST(l_tax AS DOUBLE), CAST(l_tax AS DOUBLE)), 2) AS ATAN2_TAX FROM '{lineitem}' LIMIT 10;" }, diff --git a/substrait_consumer/testdata/function/arithmetic/scalar/cos.json b/substrait_consumer/testdata/function/arithmetic/scalar/cos.json index e70e773a..97ea4bc8 100644 --- a/substrait_consumer/testdata/function/arithmetic/scalar/cos.json +++ b/substrait_consumer/testdata/function/arithmetic/scalar/cos.json @@ -7,7 +7,8 @@ "producers": [ "datafusion", "duckdb", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT round(cos(CAST(ps_supplycost AS DOUBLE)), 2) AS COS_SUPPLY FROM '{partsupp}' LIMIT 10;" }, diff --git a/substrait_consumer/testdata/function/arithmetic/scalar/divide.json b/substrait_consumer/testdata/function/arithmetic/scalar/divide.json index 237bcb21..786b8aaa 100644 --- a/substrait_consumer/testdata/function/arithmetic/scalar/divide.json +++ b/substrait_consumer/testdata/function/arithmetic/scalar/divide.json @@ -7,7 +7,8 @@ "producers": [ "datafusion", "duckdb", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT PS_PARTKEY, PS_PARTKEY / 10 AS DIVIDE_KEY FROM '{partsupp}' LIMIT 10;" }, diff --git a/substrait_consumer/testdata/function/arithmetic/scalar/exp.json b/substrait_consumer/testdata/function/arithmetic/scalar/exp.json index af268641..04cd680c 100644 --- a/substrait_consumer/testdata/function/arithmetic/scalar/exp.json +++ b/substrait_consumer/testdata/function/arithmetic/scalar/exp.json @@ -7,7 +7,8 @@ "producers": [ "datafusion", "duckdb", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT PS_PARTKEY, round(exp(CAST(PS_PARTKEY AS DOUBLE)), 2) AS EXP_KEY FROM '{partsupp}' LIMIT 10;" }, diff --git a/substrait_consumer/testdata/function/arithmetic/scalar/modulus.json b/substrait_consumer/testdata/function/arithmetic/scalar/modulus.json index 5ca31380..2f9dd5a8 100644 --- a/substrait_consumer/testdata/function/arithmetic/scalar/modulus.json +++ b/substrait_consumer/testdata/function/arithmetic/scalar/modulus.json @@ -6,7 +6,8 @@ "sql_query": { "producers": [ "duckdb", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT PS_PARTKEY, mod(PS_PARTKEY, 10) AS MODULUS_KEY FROM '{partsupp}' LIMIT 10;" }, diff --git a/substrait_consumer/testdata/function/arithmetic/scalar/multiply.json b/substrait_consumer/testdata/function/arithmetic/scalar/multiply.json index 6be09466..869ef8ed 100644 --- a/substrait_consumer/testdata/function/arithmetic/scalar/multiply.json +++ b/substrait_consumer/testdata/function/arithmetic/scalar/multiply.json @@ -7,7 +7,8 @@ "producers": [ "datafusion", "duckdb", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT PS_PARTKEY, PS_PARTKEY * 10 AS MULTIPLY_KEY FROM '{partsupp}' LIMIT 10;" }, diff --git a/substrait_consumer/testdata/function/arithmetic/scalar/power.json b/substrait_consumer/testdata/function/arithmetic/scalar/power.json index 184311ab..459c9a5b 100644 --- a/substrait_consumer/testdata/function/arithmetic/scalar/power.json +++ b/substrait_consumer/testdata/function/arithmetic/scalar/power.json @@ -6,7 +6,8 @@ "sql_query": { "producers": [ "datafusion", - "duckdb" + "duckdb", + "spark" ], "query": "SELECT PS_PARTKEY, power(PS_PARTKEY, 2) AS POWER_KEY FROM '{partsupp}' LIMIT 10;" }, diff --git a/substrait_consumer/testdata/function/arithmetic/scalar/sin.json b/substrait_consumer/testdata/function/arithmetic/scalar/sin.json index 990da6b3..57f4fab1 100644 --- a/substrait_consumer/testdata/function/arithmetic/scalar/sin.json +++ b/substrait_consumer/testdata/function/arithmetic/scalar/sin.json @@ -7,7 +7,8 @@ "producers": [ "datafusion", "duckdb", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT round(sin(CAST(ps_supplycost AS DOUBLE)), 2) AS SIN_SUPPLY FROM '{partsupp}' LIMIT 10;" }, diff --git a/substrait_consumer/testdata/function/arithmetic/scalar/sqrt.json b/substrait_consumer/testdata/function/arithmetic/scalar/sqrt.json index 6cc196f2..a36b1218 100644 --- a/substrait_consumer/testdata/function/arithmetic/scalar/sqrt.json +++ b/substrait_consumer/testdata/function/arithmetic/scalar/sqrt.json @@ -7,7 +7,8 @@ "producers": [ "datafusion", "duckdb", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT PS_PARTKEY, round(sqrt(CAST(PS_PARTKEY AS DOUBLE)), 2) AS SQRT_KEY FROM '{partsupp}' LIMIT 10;" }, diff --git a/substrait_consumer/testdata/function/arithmetic/scalar/subtract.json b/substrait_consumer/testdata/function/arithmetic/scalar/subtract.json index 8c10e94b..8d79a87e 100644 --- a/substrait_consumer/testdata/function/arithmetic/scalar/subtract.json +++ b/substrait_consumer/testdata/function/arithmetic/scalar/subtract.json @@ -7,7 +7,8 @@ "producers": [ "datafusion", "duckdb", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT PS_PARTKEY, PS_SUPPKEY, PS_PARTKEY - PS_SUPPKEY AS SUBTRACT_KEY FROM '{partsupp}' LIMIT 10;" }, diff --git a/substrait_consumer/testdata/function/arithmetic/scalar/tan.json b/substrait_consumer/testdata/function/arithmetic/scalar/tan.json index 15dded46..7cdd64ef 100644 --- a/substrait_consumer/testdata/function/arithmetic/scalar/tan.json +++ b/substrait_consumer/testdata/function/arithmetic/scalar/tan.json @@ -7,7 +7,8 @@ "producers": [ "datafusion", "duckdb", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT round(tan(CAST(ps_supplycost AS DOUBLE)), 2) AS TAN_SUPPLY FROM '{partsupp}' LIMIT 10;" }, diff --git a/substrait_consumer/testdata/integration/tpch/q01.json b/substrait_consumer/testdata/integration/tpch/q01.json index c27c4ab7..9f4caf12 100644 --- a/substrait_consumer/testdata/integration/tpch/q01.json +++ b/substrait_consumer/testdata/integration/tpch/q01.json @@ -6,7 +6,8 @@ "sql_query": { "producers": [ "duckdb", - "isthmus" + "isthmus", + "spark" ], "query_path": "q01.sql" }, diff --git a/substrait_consumer/testdata/integration/tpch/q02.json b/substrait_consumer/testdata/integration/tpch/q02.json index e4338a9c..1fbc4093 100644 --- a/substrait_consumer/testdata/integration/tpch/q02.json +++ b/substrait_consumer/testdata/integration/tpch/q02.json @@ -10,7 +10,8 @@ "sql_query": { "producers": [ "duckdb", - "isthmus" + "isthmus", + "spark" ], "query_path": "q02.sql" }, diff --git a/substrait_consumer/testdata/integration/tpch/q03.json b/substrait_consumer/testdata/integration/tpch/q03.json index b033b06a..033a829a 100644 --- a/substrait_consumer/testdata/integration/tpch/q03.json +++ b/substrait_consumer/testdata/integration/tpch/q03.json @@ -8,7 +8,8 @@ "sql_query": { "producers": [ "duckdb", - "isthmus" + "isthmus", + "spark" ], "query_path": "q03.sql" }, diff --git a/substrait_consumer/testdata/integration/tpch/q04.json b/substrait_consumer/testdata/integration/tpch/q04.json index bf73028e..2b4dd018 100644 --- a/substrait_consumer/testdata/integration/tpch/q04.json +++ b/substrait_consumer/testdata/integration/tpch/q04.json @@ -7,7 +7,8 @@ "sql_query": { "producers": [ "duckdb", - "isthmus" + "isthmus", + "spark" ], "query_path": "q04.sql" }, diff --git a/substrait_consumer/testdata/integration/tpch/q05.json b/substrait_consumer/testdata/integration/tpch/q05.json index 5ebf04e8..170e9aba 100644 --- a/substrait_consumer/testdata/integration/tpch/q05.json +++ b/substrait_consumer/testdata/integration/tpch/q05.json @@ -11,7 +11,8 @@ "sql_query": { "producers": [ "duckdb", - "isthmus" + "isthmus", + "spark" ], "query_path": "q05.sql" }, diff --git a/substrait_consumer/testdata/integration/tpch/q06.json b/substrait_consumer/testdata/integration/tpch/q06.json index 01aa19d4..80b45718 100644 --- a/substrait_consumer/testdata/integration/tpch/q06.json +++ b/substrait_consumer/testdata/integration/tpch/q06.json @@ -6,7 +6,8 @@ "sql_query": { "producers": [ "duckdb", - "isthmus" + "isthmus", + "spark" ], "query_path": "q06.sql" }, diff --git a/substrait_consumer/testdata/integration/tpch/q07.json b/substrait_consumer/testdata/integration/tpch/q07.json index fe04f60f..c34e29e9 100644 --- a/substrait_consumer/testdata/integration/tpch/q07.json +++ b/substrait_consumer/testdata/integration/tpch/q07.json @@ -10,7 +10,8 @@ "sql_query": { "producers": [ "duckdb", - "isthmus" + "isthmus", + "spark" ], "query_path": "q07.sql" }, diff --git a/substrait_consumer/testdata/integration/tpch/q08.json b/substrait_consumer/testdata/integration/tpch/q08.json index eb1c5b89..4da373ce 100644 --- a/substrait_consumer/testdata/integration/tpch/q08.json +++ b/substrait_consumer/testdata/integration/tpch/q08.json @@ -12,7 +12,8 @@ "sql_query": { "producers": [ "duckdb", - "isthmus" + "isthmus", + "spark" ], "query_path": "q08.sql" }, diff --git a/substrait_consumer/testdata/integration/tpch/q09.json b/substrait_consumer/testdata/integration/tpch/q09.json index ddfbfecc..e980ef2f 100644 --- a/substrait_consumer/testdata/integration/tpch/q09.json +++ b/substrait_consumer/testdata/integration/tpch/q09.json @@ -11,7 +11,8 @@ "sql_query": { "producers": [ "duckdb", - "isthmus" + "isthmus", + "spark" ], "query_path": "q09.sql" }, diff --git a/substrait_consumer/testdata/integration/tpch/q10.json b/substrait_consumer/testdata/integration/tpch/q10.json index ff268e7d..17da27e8 100644 --- a/substrait_consumer/testdata/integration/tpch/q10.json +++ b/substrait_consumer/testdata/integration/tpch/q10.json @@ -9,7 +9,8 @@ "sql_query": { "producers": [ "duckdb", - "isthmus" + "isthmus", + "spark" ], "query_path": "q10.sql" }, diff --git a/substrait_consumer/testdata/integration/tpch/q11.json b/substrait_consumer/testdata/integration/tpch/q11.json index 66bf0aca..8d6e040e 100644 --- a/substrait_consumer/testdata/integration/tpch/q11.json +++ b/substrait_consumer/testdata/integration/tpch/q11.json @@ -8,7 +8,8 @@ "sql_query": { "producers": [ "duckdb", - "isthmus" + "isthmus", + "spark" ], "query_path": "q11.sql" }, diff --git a/substrait_consumer/testdata/integration/tpch/q12.json b/substrait_consumer/testdata/integration/tpch/q12.json index 701c84ee..e1b0e51e 100644 --- a/substrait_consumer/testdata/integration/tpch/q12.json +++ b/substrait_consumer/testdata/integration/tpch/q12.json @@ -7,7 +7,8 @@ "sql_query": { "producers": [ "duckdb", - "isthmus" + "isthmus", + "spark" ], "query_path": "q12.sql" }, diff --git a/substrait_consumer/testdata/integration/tpch/q13.json b/substrait_consumer/testdata/integration/tpch/q13.json index 46f3930e..af19f4e3 100644 --- a/substrait_consumer/testdata/integration/tpch/q13.json +++ b/substrait_consumer/testdata/integration/tpch/q13.json @@ -7,7 +7,8 @@ "sql_query": { "producers": [ "duckdb", - "isthmus" + "isthmus", + "spark" ], "query_path": "q13.sql" }, diff --git a/substrait_consumer/testdata/integration/tpch/q14.json b/substrait_consumer/testdata/integration/tpch/q14.json index fd3d3b32..69c90364 100644 --- a/substrait_consumer/testdata/integration/tpch/q14.json +++ b/substrait_consumer/testdata/integration/tpch/q14.json @@ -7,7 +7,8 @@ "sql_query": { "producers": [ "duckdb", - "isthmus" + "isthmus", + "spark" ], "query_path": "q14.sql" }, diff --git a/substrait_consumer/testdata/integration/tpch/q15.json b/substrait_consumer/testdata/integration/tpch/q15.json index 63fe0be1..aae703af 100644 --- a/substrait_consumer/testdata/integration/tpch/q15.json +++ b/substrait_consumer/testdata/integration/tpch/q15.json @@ -7,7 +7,8 @@ "sql_query": { "producers": [ "duckdb", - "isthmus" + "isthmus", + "spark" ], "query_path": "q15.sql" }, diff --git a/substrait_consumer/testdata/integration/tpch/q16.json b/substrait_consumer/testdata/integration/tpch/q16.json index 987d5cb8..cc196eb2 100644 --- a/substrait_consumer/testdata/integration/tpch/q16.json +++ b/substrait_consumer/testdata/integration/tpch/q16.json @@ -8,7 +8,8 @@ "sql_query": { "producers": [ "duckdb", - "isthmus" + "isthmus", + "spark" ], "query_path": "q16.sql" }, diff --git a/substrait_consumer/testdata/integration/tpch/q17.json b/substrait_consumer/testdata/integration/tpch/q17.json index af01a9a7..335719cc 100644 --- a/substrait_consumer/testdata/integration/tpch/q17.json +++ b/substrait_consumer/testdata/integration/tpch/q17.json @@ -7,7 +7,8 @@ "sql_query": { "producers": [ "duckdb", - "isthmus" + "isthmus", + "spark" ], "query_path": "q17.sql" }, diff --git a/substrait_consumer/testdata/integration/tpch/q19.json b/substrait_consumer/testdata/integration/tpch/q19.json index a76fd34a..a61545f2 100644 --- a/substrait_consumer/testdata/integration/tpch/q19.json +++ b/substrait_consumer/testdata/integration/tpch/q19.json @@ -7,7 +7,8 @@ "sql_query": { "producers": [ "duckdb", - "isthmus" + "isthmus", + "spark" ], "query_path": "q19.sql" }, diff --git a/substrait_consumer/testdata/integration/tpch/q20.json b/substrait_consumer/testdata/integration/tpch/q20.json index 533e0d1a..0142428e 100644 --- a/substrait_consumer/testdata/integration/tpch/q20.json +++ b/substrait_consumer/testdata/integration/tpch/q20.json @@ -10,7 +10,8 @@ "sql_query": { "producers": [ "duckdb", - "isthmus" + "isthmus", + "spark" ], "query_path": "q20.sql" }, diff --git a/substrait_consumer/testdata/integration/tpch/q21.json b/substrait_consumer/testdata/integration/tpch/q21.json index 6aa7e506..52f82735 100644 --- a/substrait_consumer/testdata/integration/tpch/q21.json +++ b/substrait_consumer/testdata/integration/tpch/q21.json @@ -9,7 +9,8 @@ "sql_query": { "producers": [ "duckdb", - "isthmus" + "isthmus", + "spark" ], "query_path": "q21.sql" }, diff --git a/substrait_consumer/testdata/integration/tpch/q22.json b/substrait_consumer/testdata/integration/tpch/q22.json index 7ea3ac6e..670e666e 100644 --- a/substrait_consumer/testdata/integration/tpch/q22.json +++ b/substrait_consumer/testdata/integration/tpch/q22.json @@ -7,7 +7,8 @@ "sql_query": { "producers": [ "duckdb", - "isthmus" + "isthmus", + "spark" ], "query_path": "q22.sql" }, diff --git a/substrait_consumer/testdata/relation/aggregate/aggregate_in_subquery.json b/substrait_consumer/testdata/relation/aggregate/aggregate_in_subquery.json index 290eeefa..974e4267 100644 --- a/substrait_consumer/testdata/relation/aggregate/aggregate_in_subquery.json +++ b/substrait_consumer/testdata/relation/aggregate/aggregate_in_subquery.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT O_TOTALPRICE FROM '{orders}' WHERE O_TOTALPRICE <= (SELECT AVG(O_TOTALPRICE) FROM '{orders}')" }, diff --git a/substrait_consumer/testdata/relation/aggregate/aggregate_with_computation.json b/substrait_consumer/testdata/relation/aggregate/aggregate_with_computation.json index 5c41ee63..3af6ce96 100644 --- a/substrait_consumer/testdata/relation/aggregate/aggregate_with_computation.json +++ b/substrait_consumer/testdata/relation/aggregate/aggregate_with_computation.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT AVG(O_TOTALPRICE) * 10 FROM '{orders}'" }, diff --git a/substrait_consumer/testdata/relation/aggregate/aggregate_with_group_by.json b/substrait_consumer/testdata/relation/aggregate/aggregate_with_group_by.json index 2942124b..de1bad46 100644 --- a/substrait_consumer/testdata/relation/aggregate/aggregate_with_group_by.json +++ b/substrait_consumer/testdata/relation/aggregate/aggregate_with_group_by.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT L_ORDERKEY, L_LINENUMBER, count(*) FROM '{lineitem}' GROUP BY L_ORDERKEY, L_LINENUMBER ORDER BY L_ORDERKEY, L_LINENUMBER" }, diff --git a/substrait_consumer/testdata/relation/aggregate/aggregate_with_group_by_cube.json b/substrait_consumer/testdata/relation/aggregate/aggregate_with_group_by_cube.json index 7eee4cf8..43aa0971 100644 --- a/substrait_consumer/testdata/relation/aggregate/aggregate_with_group_by_cube.json +++ b/substrait_consumer/testdata/relation/aggregate/aggregate_with_group_by_cube.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT L_ORDERKEY, L_LINENUMBER, count(*) FROM '{lineitem}' GROUP BY CUBE(L_ORDERKEY, L_LINENUMBER) ORDER BY L_ORDERKEY, L_LINENUMBER" }, diff --git a/substrait_consumer/testdata/relation/aggregate/aggregate_with_group_by_rollup.json b/substrait_consumer/testdata/relation/aggregate/aggregate_with_group_by_rollup.json index 57c5aa37..4ce6408d 100644 --- a/substrait_consumer/testdata/relation/aggregate/aggregate_with_group_by_rollup.json +++ b/substrait_consumer/testdata/relation/aggregate/aggregate_with_group_by_rollup.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT L_ORDERKEY, L_LINENUMBER, count(*) FROM '{lineitem}' GROUP BY ROLLUP(L_ORDERKEY, L_LINENUMBER) ORDER BY L_ORDERKEY, L_LINENUMBER" }, diff --git a/substrait_consumer/testdata/relation/aggregate/aggregate_with_grouping_set.json b/substrait_consumer/testdata/relation/aggregate/aggregate_with_grouping_set.json index 35145fb5..a605f61c 100644 --- a/substrait_consumer/testdata/relation/aggregate/aggregate_with_grouping_set.json +++ b/substrait_consumer/testdata/relation/aggregate/aggregate_with_grouping_set.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT SUM(L_EXTENDEDPRICE), L_LINENUMBER, L_ORDERKEY FROM '{lineitem}' GROUP BY GROUPING SETS ( (L_LINENUMBER), (L_ORDERKEY) ) ORDER BY L_LINENUMBER" }, diff --git a/substrait_consumer/testdata/relation/aggregate/computation_between_aggregates.json b/substrait_consumer/testdata/relation/aggregate/computation_between_aggregates.json index 08cb678c..26b06c27 100644 --- a/substrait_consumer/testdata/relation/aggregate/computation_between_aggregates.json +++ b/substrait_consumer/testdata/relation/aggregate/computation_between_aggregates.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT AVG(O_TOTALPRICE) + MAX(O_TOTALPRICE) FROM '{orders}'" }, diff --git a/substrait_consumer/testdata/relation/aggregate/compute_within_aggregate.json b/substrait_consumer/testdata/relation/aggregate/compute_within_aggregate.json index 0e4ea67b..bf7ff0ed 100644 --- a/substrait_consumer/testdata/relation/aggregate/compute_within_aggregate.json +++ b/substrait_consumer/testdata/relation/aggregate/compute_within_aggregate.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT AVG(O_TOTALPRICE * 10) FROM '{orders}'" }, diff --git a/substrait_consumer/testdata/relation/aggregate/multiple_measure_aggregate.json b/substrait_consumer/testdata/relation/aggregate/multiple_measure_aggregate.json index e7af2ad8..3502dc03 100644 --- a/substrait_consumer/testdata/relation/aggregate/multiple_measure_aggregate.json +++ b/substrait_consumer/testdata/relation/aggregate/multiple_measure_aggregate.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT MIN(O_TOTALPRICE), MAX(O_TOTALPRICE), AVG(O_TOTALPRICE) FROM '{orders}'" }, diff --git a/substrait_consumer/testdata/relation/aggregate/single_measure_aggregate.json b/substrait_consumer/testdata/relation/aggregate/single_measure_aggregate.json index 674eccdf..9530f5e6 100644 --- a/substrait_consumer/testdata/relation/aggregate/single_measure_aggregate.json +++ b/substrait_consumer/testdata/relation/aggregate/single_measure_aggregate.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT COUNT(L_PARTKEY) FROM '{lineitem}'" }, diff --git a/substrait_consumer/testdata/relation/ddl/create_view.json b/substrait_consumer/testdata/relation/ddl/create_view.json index 7fba1c50..519bbc91 100644 --- a/substrait_consumer/testdata/relation/ddl/create_view.json +++ b/substrait_consumer/testdata/relation/ddl/create_view.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "CREATE VIEW customer_view AS SELECT C_CUSTKEY, C_NAME, FROM '{customer}';" }, diff --git a/substrait_consumer/testdata/relation/filter/having.json b/substrait_consumer/testdata/relation/filter/having.json index 45ff5ffd..b4f18687 100644 --- a/substrait_consumer/testdata/relation/filter/having.json +++ b/substrait_consumer/testdata/relation/filter/having.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT L_QUANTITY, COUNT(*) FROM '{lineitem}' GROUP BY L_QUANTITY HAVING COUNT(*) > 12100 ORDER BY L_QUANTITY" }, diff --git a/substrait_consumer/testdata/relation/filter/where_and.json b/substrait_consumer/testdata/relation/filter/where_and.json index 9aa59e81..ac77da0e 100644 --- a/substrait_consumer/testdata/relation/filter/where_and.json +++ b/substrait_consumer/testdata/relation/filter/where_and.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT L_ORDERKEY, L_SHIPINSTRUCT FROM '{lineitem}' WHERE L_ORDERKEY = 2 AND L_SHIPINSTRUCT = 'TAKE BACK RETURN'" }, diff --git a/substrait_consumer/testdata/relation/filter/where_between.json b/substrait_consumer/testdata/relation/filter/where_between.json index 93320da0..e06f3279 100644 --- a/substrait_consumer/testdata/relation/filter/where_between.json +++ b/substrait_consumer/testdata/relation/filter/where_between.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT L_ORDERKEY FROM '{lineitem}' WHERE L_ORDERKEY BETWEEN 20 AND 50 LIMIT 20;" }, diff --git a/substrait_consumer/testdata/relation/filter/where_equal_multi_col.json b/substrait_consumer/testdata/relation/filter/where_equal_multi_col.json index d1d0b877..eac3a790 100644 --- a/substrait_consumer/testdata/relation/filter/where_equal_multi_col.json +++ b/substrait_consumer/testdata/relation/filter/where_equal_multi_col.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT L_DISCOUNT, L_TAX FROM '{lineitem}' WHERE L_DISCOUNT = L_TAX ORDER BY L_DISCOUNT LIMIT 20;" }, diff --git a/substrait_consumer/testdata/relation/filter/where_gt_multi_col.json b/substrait_consumer/testdata/relation/filter/where_gt_multi_col.json index 3b806d44..1fb473c0 100644 --- a/substrait_consumer/testdata/relation/filter/where_gt_multi_col.json +++ b/substrait_consumer/testdata/relation/filter/where_gt_multi_col.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT L_DISCOUNT, L_TAX FROM '{lineitem}' WHERE L_DISCOUNT > L_TAX ORDER BY L_DISCOUNT, L_TAX LIMIT 20;" }, diff --git a/substrait_consumer/testdata/relation/filter/where_gte_multi_col.json b/substrait_consumer/testdata/relation/filter/where_gte_multi_col.json index 92daeff2..73cc80a5 100644 --- a/substrait_consumer/testdata/relation/filter/where_gte_multi_col.json +++ b/substrait_consumer/testdata/relation/filter/where_gte_multi_col.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT L_DISCOUNT, L_TAX FROM '{lineitem}' WHERE L_DISCOUNT >= L_TAX ORDER BY L_DISCOUNT, L_TAX LIMIT 20;" }, diff --git a/substrait_consumer/testdata/relation/filter/where_in.json b/substrait_consumer/testdata/relation/filter/where_in.json index c93c4cfc..dc3527cd 100644 --- a/substrait_consumer/testdata/relation/filter/where_in.json +++ b/substrait_consumer/testdata/relation/filter/where_in.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT L_ORDERKEY FROM '{lineitem}' WHERE L_ORDERKEY IN (1, 2, 3)" }, diff --git a/substrait_consumer/testdata/relation/filter/where_like.json b/substrait_consumer/testdata/relation/filter/where_like.json index 651c3e02..138fbf99 100644 --- a/substrait_consumer/testdata/relation/filter/where_like.json +++ b/substrait_consumer/testdata/relation/filter/where_like.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT L_SHIPINSTRUCT, L_ORDERKEY FROM '{lineitem}' WHERE L_SHIPINSTRUCT LIKE '%DELIVER IN PERSON%' ORDER BY L_ORDERKEY LIMIT 20;" }, diff --git a/substrait_consumer/testdata/relation/filter/where_lt_multi_col.json b/substrait_consumer/testdata/relation/filter/where_lt_multi_col.json index 724208d2..b65919c2 100644 --- a/substrait_consumer/testdata/relation/filter/where_lt_multi_col.json +++ b/substrait_consumer/testdata/relation/filter/where_lt_multi_col.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT L_DISCOUNT, L_TAX FROM '{lineitem}' WHERE L_DISCOUNT < L_TAX ORDER BY L_DISCOUNT, L_TAX LIMIT 20;" }, diff --git a/substrait_consumer/testdata/relation/filter/where_lte_multi_col.json b/substrait_consumer/testdata/relation/filter/where_lte_multi_col.json index 1459f6f8..464c6549 100644 --- a/substrait_consumer/testdata/relation/filter/where_lte_multi_col.json +++ b/substrait_consumer/testdata/relation/filter/where_lte_multi_col.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT L_DISCOUNT, L_TAX FROM '{lineitem}' WHERE L_DISCOUNT <= L_TAX ORDER BY L_DISCOUNT, L_TAX LIMIT 20;" }, diff --git a/substrait_consumer/testdata/relation/filter/where_not_equal_multi_col.json b/substrait_consumer/testdata/relation/filter/where_not_equal_multi_col.json index db3f69e7..9cf0734b 100644 --- a/substrait_consumer/testdata/relation/filter/where_not_equal_multi_col.json +++ b/substrait_consumer/testdata/relation/filter/where_not_equal_multi_col.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT L_DISCOUNT, L_TAX FROM '{lineitem}' WHERE L_DISCOUNT != L_TAX ORDER BY L_DISCOUNT, L_TAX LIMIT 20;" }, diff --git a/substrait_consumer/testdata/relation/filter/where_or.json b/substrait_consumer/testdata/relation/filter/where_or.json index 2ce94f53..6c1b9adb 100644 --- a/substrait_consumer/testdata/relation/filter/where_or.json +++ b/substrait_consumer/testdata/relation/filter/where_or.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT L_ORDERKEY, L_SHIPINSTRUCT FROM '{lineitem}' WHERE L_ORDERKEY = 2 OR L_ORDERKEY = 3" }, diff --git a/substrait_consumer/testdata/relation/join/cross_join.json b/substrait_consumer/testdata/relation/join/cross_join.json index fca0b1f9..6408a61a 100644 --- a/substrait_consumer/testdata/relation/join/cross_join.json +++ b/substrait_consumer/testdata/relation/join/cross_join.json @@ -8,7 +8,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT c.C_CUSTKEY, c.C_NAME, o.O_ORDERKEY FROM '{customer}' c CROSS JOIN '{orders}' o" }, diff --git a/substrait_consumer/testdata/relation/join/full_join.json b/substrait_consumer/testdata/relation/join/full_join.json index a2808f6a..45c79156 100644 --- a/substrait_consumer/testdata/relation/join/full_join.json +++ b/substrait_consumer/testdata/relation/join/full_join.json @@ -8,7 +8,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT c.C_CUSTKEY, c.C_NAME, o.O_ORDERKEY FROM '{customer}' c FULL JOIN '{orders}' o ON c.C_CUSTKEY = o.O_CUSTKEY;" }, diff --git a/substrait_consumer/testdata/relation/join/inner_join.json b/substrait_consumer/testdata/relation/join/inner_join.json index 7e6a0b77..779c9845 100644 --- a/substrait_consumer/testdata/relation/join/inner_join.json +++ b/substrait_consumer/testdata/relation/join/inner_join.json @@ -8,7 +8,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT c.C_CUSTKEY, c.C_NAME, o.O_ORDERKEY FROM '{customer}' c INNER JOIN '{orders}' o ON c.C_CUSTKEY = o.O_CUSTKEY;" }, diff --git a/substrait_consumer/testdata/relation/join/left_anti_join.json b/substrait_consumer/testdata/relation/join/left_anti_join.json index b2db8bad..c13a4e1e 100644 --- a/substrait_consumer/testdata/relation/join/left_anti_join.json +++ b/substrait_consumer/testdata/relation/join/left_anti_join.json @@ -8,7 +8,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT c.C_CUSTKEY, c.C_NAME FROM '{customer}' c WHERE NOT EXISTS ( SELECT 1 FROM '{orders}' o WHERE o.O_CUSTKEY = c.C_CUSTKEY );" }, diff --git a/substrait_consumer/testdata/relation/join/left_join.json b/substrait_consumer/testdata/relation/join/left_join.json index 4a2b5cd2..5f3fba87 100644 --- a/substrait_consumer/testdata/relation/join/left_join.json +++ b/substrait_consumer/testdata/relation/join/left_join.json @@ -8,7 +8,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT c.C_CUSTKEY, c.C_NAME, o.O_ORDERKEY FROM '{customer}' c LEFT JOIN '{orders}' o ON c.C_CUSTKEY = o.O_CUSTKEY;" }, diff --git a/substrait_consumer/testdata/relation/join/left_semi_join.json b/substrait_consumer/testdata/relation/join/left_semi_join.json index c55f5628..2a428587 100644 --- a/substrait_consumer/testdata/relation/join/left_semi_join.json +++ b/substrait_consumer/testdata/relation/join/left_semi_join.json @@ -8,7 +8,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT c.C_CUSTKEY, c.C_NAME FROM '{customer}' c WHERE EXISTS ( SELECT 1 FROM '{orders}' o WHERE o.O_CUSTKEY = c.C_CUSTKEY );" }, diff --git a/substrait_consumer/testdata/relation/join/left_single_join.json b/substrait_consumer/testdata/relation/join/left_single_join.json index 44789017..3d7e0d82 100644 --- a/substrait_consumer/testdata/relation/join/left_single_join.json +++ b/substrait_consumer/testdata/relation/join/left_single_join.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT c1.C_CUSTKEY AS c1key, c1.C_NAME AS c1name, c1.C_NATIONKEY AS c1nationakey, c2.C_CUSTKEY AS c2key, c2.C_NAME AS c2name, c2.C_NATIONKEY AS c2nationakey FROM '{customer}' c1 LEFT JOIN '{customer}' c2 ON c1.C_NATIONKEY = c2.C_NATIONKEY AND c1.C_CUSTKEY <> c2.C_CUSTKEY;" }, diff --git a/substrait_consumer/testdata/relation/join/right_anti_join.json b/substrait_consumer/testdata/relation/join/right_anti_join.json index 8efd287d..f3621556 100644 --- a/substrait_consumer/testdata/relation/join/right_anti_join.json +++ b/substrait_consumer/testdata/relation/join/right_anti_join.json @@ -8,7 +8,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT o.O_ORDERKEY, o.O_CUSTKEY FROM '{orders}' o WHERE NOT EXISTS ( SELECT 1 FROM '{lineitem}' l WHERE l.L_ORDERKEY = o.O_ORDERKEY );" }, diff --git a/substrait_consumer/testdata/relation/join/right_join.json b/substrait_consumer/testdata/relation/join/right_join.json index 96146f77..7bfdcdf9 100644 --- a/substrait_consumer/testdata/relation/join/right_join.json +++ b/substrait_consumer/testdata/relation/join/right_join.json @@ -8,7 +8,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT c.C_CUSTKEY, c.C_NAME, o.O_ORDERKEY FROM '{customer}' c RIGHT JOIN '{orders}' o ON c.C_CUSTKEY = o.O_CUSTKEY;" }, diff --git a/substrait_consumer/testdata/relation/join/right_semi_join.json b/substrait_consumer/testdata/relation/join/right_semi_join.json index a18391e7..f086bdec 100644 --- a/substrait_consumer/testdata/relation/join/right_semi_join.json +++ b/substrait_consumer/testdata/relation/join/right_semi_join.json @@ -8,7 +8,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT o.O_ORDERKEY, o.O_CUSTKEY FROM '{orders}' o WHERE EXISTS ( SELECT 1 FROM '{customer}' c WHERE c.C_CUSTKEY = o.O_CUSTKEY );" }, diff --git a/substrait_consumer/testdata/relation/join/right_single_join.json b/substrait_consumer/testdata/relation/join/right_single_join.json index fed8bae7..c8b9bd92 100644 --- a/substrait_consumer/testdata/relation/join/right_single_join.json +++ b/substrait_consumer/testdata/relation/join/right_single_join.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT c1.C_CUSTKEY AS c1key, c1.C_NAME AS c1name, c1.C_NATIONKEY AS c1nationakey, c2.C_CUSTKEY AS c2key, c2.C_NAME AS c2name, c2.C_NATIONKEY AS c2nationakey FROM '{customer}' c1 RIGHT JOIN '{customer}' c2 ON c1.C_NATIONKEY = c2.C_NATIONKEY AND c1.C_CUSTKEY <> c2.C_CUSTKEY;" }, diff --git a/substrait_consumer/testdata/relation/project/count_distinct_in_project.json b/substrait_consumer/testdata/relation/project/count_distinct_in_project.json index 47396559..1e8b0cc5 100644 --- a/substrait_consumer/testdata/relation/project/count_distinct_in_project.json +++ b/substrait_consumer/testdata/relation/project/count_distinct_in_project.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT COUNT(DISTINCT L_EXTENDEDPRICE) FROM '{lineitem}'" }, diff --git a/substrait_consumer/testdata/relation/project/distinct_in_project.json b/substrait_consumer/testdata/relation/project/distinct_in_project.json index 04c70f0a..f9c4d595 100644 --- a/substrait_consumer/testdata/relation/project/distinct_in_project.json +++ b/substrait_consumer/testdata/relation/project/distinct_in_project.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT DISTINCT L_LINESTATUS FROM '{lineitem}'" }, diff --git a/substrait_consumer/testdata/relation/project/extended_project.json b/substrait_consumer/testdata/relation/project/extended_project.json index 980e6bf2..286627b1 100644 --- a/substrait_consumer/testdata/relation/project/extended_project.json +++ b/substrait_consumer/testdata/relation/project/extended_project.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT L_QUANTITY, L_EXTENDEDPRICE*10 AS MULTI_PRICE FROM '{lineitem}'" }, diff --git a/substrait_consumer/testdata/relation/project/project_all_col.json b/substrait_consumer/testdata/relation/project/project_all_col.json index 0909db14..c41167e3 100644 --- a/substrait_consumer/testdata/relation/project/project_all_col.json +++ b/substrait_consumer/testdata/relation/project/project_all_col.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT * FROM '{region}'" }, diff --git a/substrait_consumer/testdata/relation/project/project_multi_col.json b/substrait_consumer/testdata/relation/project/project_multi_col.json index 910399dc..a2a88ab3 100644 --- a/substrait_consumer/testdata/relation/project/project_multi_col.json +++ b/substrait_consumer/testdata/relation/project/project_multi_col.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT L_DISCOUNT, L_TAX FROM '{lineitem}'" }, diff --git a/substrait_consumer/testdata/relation/project/project_single_col.json b/substrait_consumer/testdata/relation/project/project_single_col.json index 84f19ef2..a5999006 100644 --- a/substrait_consumer/testdata/relation/project/project_single_col.json +++ b/substrait_consumer/testdata/relation/project/project_single_col.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT * FROM '{lineitem}'" }, diff --git a/substrait_consumer/testdata/relation/project/subquery_in_project.json b/substrait_consumer/testdata/relation/project/subquery_in_project.json index 55ce52e3..f9e659c5 100644 --- a/substrait_consumer/testdata/relation/project/subquery_in_project.json +++ b/substrait_consumer/testdata/relation/project/subquery_in_project.json @@ -8,7 +8,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT C_CUSTKEY, (SELECT SUM(O_TOTALPRICE) FROM '{orders}' WHERE C_CUSTKEY = O_CUSTKEY) AS total_price FROM '{customer}'" }, diff --git a/substrait_consumer/testdata/relation/read/read_named_table.json b/substrait_consumer/testdata/relation/read/read_named_table.json index 3dc1394e..c4d14430 100644 --- a/substrait_consumer/testdata/relation/read/read_named_table.json +++ b/substrait_consumer/testdata/relation/read/read_named_table.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT PS_PARTKEY FROM '{partsupp}'" }, diff --git a/substrait_consumer/testdata/relation/set/except.json b/substrait_consumer/testdata/relation/set/except.json index 21494470..e25b6cda 100644 --- a/substrait_consumer/testdata/relation/set/except.json +++ b/substrait_consumer/testdata/relation/set/except.json @@ -8,7 +8,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT o_totalprice FROM '{orders}' EXCEPT SELECT c_acctbal FROM '{customer}'" }, diff --git a/substrait_consumer/testdata/relation/set/intersect.json b/substrait_consumer/testdata/relation/set/intersect.json index 84c63435..2511cb61 100644 --- a/substrait_consumer/testdata/relation/set/intersect.json +++ b/substrait_consumer/testdata/relation/set/intersect.json @@ -8,7 +8,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT C_NATIONKEY FROM '{customer}' INTERSECT SELECT N_NATIONKEY FROM '{nation}'" }, diff --git a/substrait_consumer/testdata/relation/set/union_all.json b/substrait_consumer/testdata/relation/set/union_all.json index b97fa630..038caaa4 100644 --- a/substrait_consumer/testdata/relation/set/union_all.json +++ b/substrait_consumer/testdata/relation/set/union_all.json @@ -8,7 +8,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT C_NATIONKEY FROM '{customer}' UNION ALL SELECT N_NATIONKEY FROM '{nation}'" }, diff --git a/substrait_consumer/testdata/relation/set/union_distinct.json b/substrait_consumer/testdata/relation/set/union_distinct.json index 8379a38f..d993acc6 100644 --- a/substrait_consumer/testdata/relation/set/union_distinct.json +++ b/substrait_consumer/testdata/relation/set/union_distinct.json @@ -8,7 +8,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT C_NATIONKEY FROM '{customer}' UNION SELECT N_NATIONKEY FROM '{nation}' ORDER BY C_NATIONKEY" }, diff --git a/substrait_consumer/testdata/relation/sort/multi_col_asc.json b/substrait_consumer/testdata/relation/sort/multi_col_asc.json index c8f82a7c..f4c48eb2 100644 --- a/substrait_consumer/testdata/relation/sort/multi_col_asc.json +++ b/substrait_consumer/testdata/relation/sort/multi_col_asc.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT PS_SUPPLYCOST, PS_AVAILQTY FROM '{partsupp}' ORDER BY PS_SUPPLYCOST ASC, PS_AVAILQTY LIMIT 10;" }, diff --git a/substrait_consumer/testdata/relation/sort/multi_col_asc_desc.json b/substrait_consumer/testdata/relation/sort/multi_col_asc_desc.json index 6618168e..e35ac001 100644 --- a/substrait_consumer/testdata/relation/sort/multi_col_asc_desc.json +++ b/substrait_consumer/testdata/relation/sort/multi_col_asc_desc.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT PS_SUPPLYCOST, PS_AVAILQTY FROM '{partsupp}' ORDER BY PS_SUPPLYCOST ASC, PS_AVAILQTY DESC LIMIT 10;" }, diff --git a/substrait_consumer/testdata/relation/sort/multi_col_desc.json b/substrait_consumer/testdata/relation/sort/multi_col_desc.json index d9e11448..e4aaf5d3 100644 --- a/substrait_consumer/testdata/relation/sort/multi_col_desc.json +++ b/substrait_consumer/testdata/relation/sort/multi_col_desc.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT PS_SUPPLYCOST FROM '{partsupp}' ORDER BY PS_SUPPLYCOST DESC LIMIT 10;" }, diff --git a/substrait_consumer/testdata/relation/sort/multi_col_desc_asc.json b/substrait_consumer/testdata/relation/sort/multi_col_desc_asc.json index 64b68c45..6862e30f 100644 --- a/substrait_consumer/testdata/relation/sort/multi_col_desc_asc.json +++ b/substrait_consumer/testdata/relation/sort/multi_col_desc_asc.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT PS_SUPPLYCOST, PS_AVAILQTY FROM '{partsupp}' ORDER BY PS_SUPPLYCOST DESC, PS_AVAILQTY ASC LIMIT 10;" }, diff --git a/substrait_consumer/testdata/relation/sort/order_by_col_number.json b/substrait_consumer/testdata/relation/sort/order_by_col_number.json index b8a8580b..7f2c9ca3 100644 --- a/substrait_consumer/testdata/relation/sort/order_by_col_number.json +++ b/substrait_consumer/testdata/relation/sort/order_by_col_number.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT PS_SUPPLYCOST, PS_AVAILQTY FROM '{partsupp}' ORDER BY 1, 2 LIMIT 10;" }, diff --git a/substrait_consumer/testdata/relation/sort/single_col_asc.json b/substrait_consumer/testdata/relation/sort/single_col_asc.json index 0162622d..7224923a 100644 --- a/substrait_consumer/testdata/relation/sort/single_col_asc.json +++ b/substrait_consumer/testdata/relation/sort/single_col_asc.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT PS_SUPPLYCOST FROM '{partsupp}' ORDER BY PS_SUPPLYCOST ASC LIMIT 10;" }, diff --git a/substrait_consumer/testdata/relation/sort/single_col_default_sort.json b/substrait_consumer/testdata/relation/sort/single_col_default_sort.json index cd31e17b..2cfc76c2 100644 --- a/substrait_consumer/testdata/relation/sort/single_col_default_sort.json +++ b/substrait_consumer/testdata/relation/sort/single_col_default_sort.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT PS_AVAILQTY FROM '{partsupp}' ORDER BY PS_AVAILQTY LIMIT 10;" }, diff --git a/substrait_consumer/testdata/relation/sort/single_col_desc.json b/substrait_consumer/testdata/relation/sort/single_col_desc.json index 24278eef..cbb7be90 100644 --- a/substrait_consumer/testdata/relation/sort/single_col_desc.json +++ b/substrait_consumer/testdata/relation/sort/single_col_desc.json @@ -7,7 +7,8 @@ "producers": [ "duckdb", "datafusion", - "isthmus" + "isthmus", + "spark" ], "query": "SELECT PS_SUPPLYCOST FROM '{partsupp}' ORDER BY PS_SUPPLYCOST DESC LIMIT 10;" },