diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index 9c9a657bc6e9f..aca2f163e8124 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -8167,12 +8167,12 @@ }, "SET_OPERATION_ON_MAP_TYPE" : { "message" : [ - "Cannot have MAP type columns in DataFrame which calls set operations (INTERSECT, EXCEPT, etc.), but the type of column is ." + "Cannot have MAP type columns in a set operation (INTERSECT, EXCEPT, etc.), but the type of column is ." ] }, "SET_OPERATION_ON_VARIANT_TYPE" : { "message" : [ - "Cannot have VARIANT type columns in DataFrame which calls set operations (INTERSECT, EXCEPT, etc.), but the type of column is ." + "Cannot have VARIANT type columns in a set operation (INTERSECT, EXCEPT, etc.), but the type of column is ." ] }, "SET_PATH_WHEN_DISABLED" : { diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/set-operations-unsupported-types.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/set-operations-unsupported-types.sql.out new file mode 100644 index 0000000000000..d785d33e2df39 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/set-operations-unsupported-types.sql.out @@ -0,0 +1,246 @@ +-- Automatically generated by SQLQueryTestSuite +-- !query +create temporary view map_view as select map(1, 2) as m, id from range(2) +-- !query analysis +CreateViewCommand `map_view`, select map(1, 2) as m, id from range(2), false, false, LocalTempView, UNSUPPORTED, true + +- Project [map(1, 2) AS m#x, id#xL] + +- Range (0, 2, step=1) + + +-- !query +create temporary view variant_view as select parse_json('1') as v, id from range(2) +-- !query analysis +CreateViewCommand `variant_view`, select parse_json('1') as v, id from range(2), false, false, LocalTempView, UNSUPPORTED, true + +- Project [parse_json(1, true) AS v#x, id#xL] + +- Range (0, 2, step=1) + + +-- !query +SELECT * FROM map_view INTERSECT SELECT * FROM map_view +-- !query analysis +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE", + "sqlState" : "0A000", + "messageParameters" : { + "colName" : "`m`", + "dataType" : "\"MAP\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 55, + "fragment" : "SELECT * FROM map_view INTERSECT SELECT * FROM map_view" + } ] +} + + +-- !query +SELECT * FROM map_view EXCEPT SELECT * FROM map_view +-- !query analysis +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE", + "sqlState" : "0A000", + "messageParameters" : { + "colName" : "`m`", + "dataType" : "\"MAP\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 52, + "fragment" : "SELECT * FROM map_view EXCEPT SELECT * FROM map_view" + } ] +} + + +-- !query +SELECT * FROM map_view UNION SELECT * FROM map_view +-- !query analysis +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE", + "sqlState" : "0A000", + "messageParameters" : { + "colName" : "`m`", + "dataType" : "\"MAP\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 51, + "fragment" : "SELECT * FROM map_view UNION SELECT * FROM map_view" + } ] +} + + +-- !query +SELECT DISTINCT m FROM map_view +-- !query analysis +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE", + "sqlState" : "0A000", + "messageParameters" : { + "colName" : "`m`", + "dataType" : "\"MAP\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 31, + "fragment" : "SELECT DISTINCT m FROM map_view" + } ] +} + + +-- !query +SELECT * FROM variant_view INTERSECT SELECT * FROM variant_view +-- !query analysis +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE", + "sqlState" : "0A000", + "messageParameters" : { + "colName" : "`v`", + "dataType" : "\"VARIANT\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 63, + "fragment" : "SELECT * FROM variant_view INTERSECT SELECT * FROM variant_view" + } ] +} + + +-- !query +SELECT * FROM variant_view EXCEPT SELECT * FROM variant_view +-- !query analysis +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE", + "sqlState" : "0A000", + "messageParameters" : { + "colName" : "`v`", + "dataType" : "\"VARIANT\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 60, + "fragment" : "SELECT * FROM variant_view EXCEPT SELECT * FROM variant_view" + } ] +} + + +-- !query +SELECT * FROM variant_view UNION SELECT * FROM variant_view +-- !query analysis +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE", + "sqlState" : "0A000", + "messageParameters" : { + "colName" : "`v`", + "dataType" : "\"VARIANT\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 59, + "fragment" : "SELECT * FROM variant_view UNION SELECT * FROM variant_view" + } ] +} + + +-- !query +SELECT DISTINCT v FROM variant_view +-- !query analysis +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE", + "sqlState" : "0A000", + "messageParameters" : { + "colName" : "`v`", + "dataType" : "\"VARIANT\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 35, + "fragment" : "SELECT DISTINCT v FROM variant_view" + } ] +} + + +-- !query +SELECT DISTINCT struct(v) FROM variant_view +-- !query analysis +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE", + "sqlState" : "0A000", + "messageParameters" : { + "colName" : "`struct(v)`", + "dataType" : "\"STRUCT\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 43, + "fragment" : "SELECT DISTINCT struct(v) FROM variant_view" + } ] +} + + +-- !query +SELECT DISTINCT array(v) FROM variant_view +-- !query analysis +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE", + "sqlState" : "0A000", + "messageParameters" : { + "colName" : "`array(v)`", + "dataType" : "\"ARRAY\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 42, + "fragment" : "SELECT DISTINCT array(v) FROM variant_view" + } ] +} + + +-- !query +SELECT DISTINCT map('m', v) FROM variant_view +-- !query analysis +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE", + "sqlState" : "0A000", + "messageParameters" : { + "colName" : "`map(m, v)`", + "dataType" : "\"MAP\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 45, + "fragment" : "SELECT DISTINCT map('m', v) FROM variant_view" + } ] +} diff --git a/sql/core/src/test/resources/sql-tests/inputs/set-operations-unsupported-types.sql b/sql/core/src/test/resources/sql-tests/inputs/set-operations-unsupported-types.sql new file mode 100644 index 0000000000000..8e9ef6af63e6c --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/inputs/set-operations-unsupported-types.sql @@ -0,0 +1,36 @@ +-- Tests that set operations (and the DISTINCT/de-duplication they rely on) are +-- rejected when a column has MAP or VARIANT type, since those types are not +-- orderable. See UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE and +-- UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE. + +create temporary view map_view as select map(1, 2) as m, id from range(2); + +create temporary view variant_view as select parse_json('1') as v, id from range(2); + + +-- MAP type columns in set operations +SELECT * FROM map_view INTERSECT SELECT * FROM map_view; + +SELECT * FROM map_view EXCEPT SELECT * FROM map_view; + +SELECT * FROM map_view UNION SELECT * FROM map_view; + +SELECT DISTINCT m FROM map_view; + + +-- VARIANT type columns in set operations +SELECT * FROM variant_view INTERSECT SELECT * FROM variant_view; + +SELECT * FROM variant_view EXCEPT SELECT * FROM variant_view; + +SELECT * FROM variant_view UNION SELECT * FROM variant_view; + +SELECT DISTINCT v FROM variant_view; + + +-- VARIANT nested inside complex types is also rejected +SELECT DISTINCT struct(v) FROM variant_view; + +SELECT DISTINCT array(v) FROM variant_view; + +SELECT DISTINCT map('m', v) FROM variant_view; diff --git a/sql/core/src/test/resources/sql-tests/results/set-operations-unsupported-types.sql.out b/sql/core/src/test/resources/sql-tests/results/set-operations-unsupported-types.sql.out new file mode 100644 index 0000000000000..80fde871121fd --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/set-operations-unsupported-types.sql.out @@ -0,0 +1,268 @@ +-- Automatically generated by SQLQueryTestSuite +-- !query +create temporary view map_view as select map(1, 2) as m, id from range(2) +-- !query schema +struct<> +-- !query output + + + +-- !query +create temporary view variant_view as select parse_json('1') as v, id from range(2) +-- !query schema +struct<> +-- !query output + + + +-- !query +SELECT * FROM map_view INTERSECT SELECT * FROM map_view +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE", + "sqlState" : "0A000", + "messageParameters" : { + "colName" : "`m`", + "dataType" : "\"MAP\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 55, + "fragment" : "SELECT * FROM map_view INTERSECT SELECT * FROM map_view" + } ] +} + + +-- !query +SELECT * FROM map_view EXCEPT SELECT * FROM map_view +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE", + "sqlState" : "0A000", + "messageParameters" : { + "colName" : "`m`", + "dataType" : "\"MAP\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 52, + "fragment" : "SELECT * FROM map_view EXCEPT SELECT * FROM map_view" + } ] +} + + +-- !query +SELECT * FROM map_view UNION SELECT * FROM map_view +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE", + "sqlState" : "0A000", + "messageParameters" : { + "colName" : "`m`", + "dataType" : "\"MAP\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 51, + "fragment" : "SELECT * FROM map_view UNION SELECT * FROM map_view" + } ] +} + + +-- !query +SELECT DISTINCT m FROM map_view +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE", + "sqlState" : "0A000", + "messageParameters" : { + "colName" : "`m`", + "dataType" : "\"MAP\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 31, + "fragment" : "SELECT DISTINCT m FROM map_view" + } ] +} + + +-- !query +SELECT * FROM variant_view INTERSECT SELECT * FROM variant_view +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE", + "sqlState" : "0A000", + "messageParameters" : { + "colName" : "`v`", + "dataType" : "\"VARIANT\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 63, + "fragment" : "SELECT * FROM variant_view INTERSECT SELECT * FROM variant_view" + } ] +} + + +-- !query +SELECT * FROM variant_view EXCEPT SELECT * FROM variant_view +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE", + "sqlState" : "0A000", + "messageParameters" : { + "colName" : "`v`", + "dataType" : "\"VARIANT\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 60, + "fragment" : "SELECT * FROM variant_view EXCEPT SELECT * FROM variant_view" + } ] +} + + +-- !query +SELECT * FROM variant_view UNION SELECT * FROM variant_view +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE", + "sqlState" : "0A000", + "messageParameters" : { + "colName" : "`v`", + "dataType" : "\"VARIANT\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 59, + "fragment" : "SELECT * FROM variant_view UNION SELECT * FROM variant_view" + } ] +} + + +-- !query +SELECT DISTINCT v FROM variant_view +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE", + "sqlState" : "0A000", + "messageParameters" : { + "colName" : "`v`", + "dataType" : "\"VARIANT\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 35, + "fragment" : "SELECT DISTINCT v FROM variant_view" + } ] +} + + +-- !query +SELECT DISTINCT struct(v) FROM variant_view +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE", + "sqlState" : "0A000", + "messageParameters" : { + "colName" : "`struct(v)`", + "dataType" : "\"STRUCT\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 43, + "fragment" : "SELECT DISTINCT struct(v) FROM variant_view" + } ] +} + + +-- !query +SELECT DISTINCT array(v) FROM variant_view +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE", + "sqlState" : "0A000", + "messageParameters" : { + "colName" : "`array(v)`", + "dataType" : "\"ARRAY\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 42, + "fragment" : "SELECT DISTINCT array(v) FROM variant_view" + } ] +} + + +-- !query +SELECT DISTINCT map('m', v) FROM variant_view +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE", + "sqlState" : "0A000", + "messageParameters" : { + "colName" : "`map(m, v)`", + "dataType" : "\"MAP\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 45, + "fragment" : "SELECT DISTINCT map('m', v) FROM variant_view" + } ] +}