[SPARK-57576] [SQL] Fix SET_OPERATION_ON_MAP_TYPE and SET_OPERATION_ON_VARIANT_TYPE exception messages

common/utils/src/main/resources/error/error-conditions.json

@@ -8167,12 +8167,12 @@
       },
       "SET_OPERATION_ON_MAP_TYPE" : {
         "message" : [
-          "Cannot have MAP type columns in DataFrame which calls set operations (INTERSECT, EXCEPT, etc.), but the type of column <colName> is <dataType>."
+          "Cannot have MAP type columns in a set operation (INTERSECT, EXCEPT, etc.), but the type of column <colName> is <dataType>."
         ]
       },
       "SET_OPERATION_ON_VARIANT_TYPE" : {
         "message" : [
-          "Cannot have VARIANT type columns in DataFrame which calls set operations (INTERSECT, EXCEPT, etc.), but the type of column <colName> is <dataType>."
+          "Cannot have VARIANT type columns in a set operation (INTERSECT, EXCEPT, etc.), but the type of column <colName> is <dataType>."
         ]
       },
       "SET_PATH_WHEN_DISABLED" : {

sql/core/src/test/resources/sql-tests/analyzer-results/set-operations-unsupported-types.sql.out

@@ -0,0 +1,246 @@
+-- Automatically generated by SQLQueryTestSuite
+-- !query
+create temporary view map_view as select map(1, 2) as m, id from range(2)
+-- !query analysis
+CreateViewCommand `map_view`, select map(1, 2) as m, id from range(2), false, false, LocalTempView, UNSUPPORTED, true
+   +- Project [map(1, 2) AS m#x, id#xL]
+      +- Range (0, 2, step=1)
+
+
+-- !query
+create temporary view variant_view as select parse_json('1') as v, id from range(2)
+-- !query analysis
+CreateViewCommand `variant_view`, select parse_json('1') as v, id from range(2), false, false, LocalTempView, UNSUPPORTED, true
+   +- Project [parse_json(1, true) AS v#x, id#xL]
+      +- Range (0, 2, step=1)
+
+
+-- !query
+SELECT * FROM map_view INTERSECT SELECT * FROM map_view
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+  "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE",
+  "sqlState" : "0A000",
+  "messageParameters" : {
+    "colName" : "`m`",
+    "dataType" : "\"MAP<INT, INT>\""
+  },
+  "queryContext" : [ {
+    "objectType" : "",
+    "objectName" : "",
+    "startIndex" : 1,
+    "stopIndex" : 55,
+    "fragment" : "SELECT * FROM map_view INTERSECT SELECT * FROM map_view"
+  } ]
+}
+
+
+-- !query
+SELECT * FROM map_view EXCEPT SELECT * FROM map_view
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+  "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE",
+  "sqlState" : "0A000",
+  "messageParameters" : {
+    "colName" : "`m`",
+    "dataType" : "\"MAP<INT, INT>\""
+  },
+  "queryContext" : [ {
+    "objectType" : "",
+    "objectName" : "",
+    "startIndex" : 1,
+    "stopIndex" : 52,
+    "fragment" : "SELECT * FROM map_view EXCEPT SELECT * FROM map_view"
+  } ]
+}
+
+
+-- !query
+SELECT * FROM map_view UNION SELECT * FROM map_view
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+  "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE",
+  "sqlState" : "0A000",
+  "messageParameters" : {
+    "colName" : "`m`",
+    "dataType" : "\"MAP<INT, INT>\""
+  },
+  "queryContext" : [ {
+    "objectType" : "",
+    "objectName" : "",
+    "startIndex" : 1,
+    "stopIndex" : 51,
+    "fragment" : "SELECT * FROM map_view UNION SELECT * FROM map_view"
+  } ]
+}
+
+
+-- !query
+SELECT DISTINCT m FROM map_view
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+  "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE",
+  "sqlState" : "0A000",
+  "messageParameters" : {
+    "colName" : "`m`",
+    "dataType" : "\"MAP<INT, INT>\""
+  },
+  "queryContext" : [ {
+    "objectType" : "",
+    "objectName" : "",
+    "startIndex" : 1,
+    "stopIndex" : 31,
+    "fragment" : "SELECT DISTINCT m FROM map_view"
+  } ]
+}
+
+
+-- !query
+SELECT * FROM variant_view INTERSECT SELECT * FROM variant_view
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+  "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE",
+  "sqlState" : "0A000",
+  "messageParameters" : {
+    "colName" : "`v`",
+    "dataType" : "\"VARIANT\""
+  },
+  "queryContext" : [ {
+    "objectType" : "",
+    "objectName" : "",
+    "startIndex" : 1,
+    "stopIndex" : 63,
+    "fragment" : "SELECT * FROM variant_view INTERSECT SELECT * FROM variant_view"
+  } ]
+}
+
+
+-- !query
+SELECT * FROM variant_view EXCEPT SELECT * FROM variant_view
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+  "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE",
+  "sqlState" : "0A000",
+  "messageParameters" : {
+    "colName" : "`v`",
+    "dataType" : "\"VARIANT\""
+  },
+  "queryContext" : [ {
+    "objectType" : "",
+    "objectName" : "",
+    "startIndex" : 1,
+    "stopIndex" : 60,
+    "fragment" : "SELECT * FROM variant_view EXCEPT SELECT * FROM variant_view"
+  } ]
+}
+
+
+-- !query
+SELECT * FROM variant_view UNION SELECT * FROM variant_view
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+  "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE",
+  "sqlState" : "0A000",
+  "messageParameters" : {
+    "colName" : "`v`",
+    "dataType" : "\"VARIANT\""
+  },
+  "queryContext" : [ {
+    "objectType" : "",
+    "objectName" : "",
+    "startIndex" : 1,
+    "stopIndex" : 59,
+    "fragment" : "SELECT * FROM variant_view UNION SELECT * FROM variant_view"
+  } ]
+}
+
+
+-- !query
+SELECT DISTINCT v FROM variant_view
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+  "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE",
+  "sqlState" : "0A000",
+  "messageParameters" : {
+    "colName" : "`v`",
+    "dataType" : "\"VARIANT\""
+  },
+  "queryContext" : [ {
+    "objectType" : "",
+    "objectName" : "",
+    "startIndex" : 1,
+    "stopIndex" : 35,
+    "fragment" : "SELECT DISTINCT v FROM variant_view"
+  } ]
+}
+
+
+-- !query
+SELECT DISTINCT struct(v) FROM variant_view
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+  "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE",
+  "sqlState" : "0A000",
+  "messageParameters" : {
+    "colName" : "`struct(v)`",
+    "dataType" : "\"STRUCT<v: VARIANT NOT NULL>\""
+  },
+  "queryContext" : [ {
+    "objectType" : "",
+    "objectName" : "",
+    "startIndex" : 1,
+    "stopIndex" : 43,
+    "fragment" : "SELECT DISTINCT struct(v) FROM variant_view"
+  } ]
+}
+
+
+-- !query
+SELECT DISTINCT array(v) FROM variant_view
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+  "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE",
+  "sqlState" : "0A000",
+  "messageParameters" : {
+    "colName" : "`array(v)`",
+    "dataType" : "\"ARRAY<VARIANT>\""
+  },
+  "queryContext" : [ {
+    "objectType" : "",
+    "objectName" : "",
+    "startIndex" : 1,
+    "stopIndex" : 42,
+    "fragment" : "SELECT DISTINCT array(v) FROM variant_view"
+  } ]
+}
+
+
+-- !query
+SELECT DISTINCT map('m', v) FROM variant_view
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+  "errorClass" : "UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE",
+  "sqlState" : "0A000",
+  "messageParameters" : {
+    "colName" : "`map(m, v)`",
+    "dataType" : "\"MAP<STRING, VARIANT>\""
+  },
+  "queryContext" : [ {
+    "objectType" : "",
+    "objectName" : "",
+    "startIndex" : 1,
+    "stopIndex" : 45,
+    "fragment" : "SELECT DISTINCT map('m', v) FROM variant_view"
+  } ]
+}

sql/core/src/test/resources/sql-tests/inputs/set-operations-unsupported-types.sql

@@ -0,0 +1,36 @@
+-- Tests that set operations (and the DISTINCT/de-duplication they rely on) are
+-- rejected when a column has MAP or VARIANT type, since those types are not
+-- orderable. See UNSUPPORTED_FEATURE.SET_OPERATION_ON_MAP_TYPE and
+-- UNSUPPORTED_FEATURE.SET_OPERATION_ON_VARIANT_TYPE.
+
+create temporary view map_view as select map(1, 2) as m, id from range(2);
+
+create temporary view variant_view as select parse_json('1') as v, id from range(2);
+
+
+-- MAP type columns in set operations
+SELECT * FROM map_view INTERSECT SELECT * FROM map_view;
+
+SELECT * FROM map_view EXCEPT SELECT * FROM map_view;
+
+SELECT * FROM map_view UNION SELECT * FROM map_view;
+
+SELECT DISTINCT m FROM map_view;
+
+
+-- VARIANT type columns in set operations
+SELECT * FROM variant_view INTERSECT SELECT * FROM variant_view;
+
+SELECT * FROM variant_view EXCEPT SELECT * FROM variant_view;
+
+SELECT * FROM variant_view UNION SELECT * FROM variant_view;
+
+SELECT DISTINCT v FROM variant_view;
+
+
+-- VARIANT nested inside complex types is also rejected
+SELECT DISTINCT struct(v) FROM variant_view;
+
+SELECT DISTINCT array(v) FROM variant_view;
+
+SELECT DISTINCT map('m', v) FROM variant_view;