From 1388fa1e7d35856b1964af3356fa4b90a220b7bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BE=8A=E5=B7=9D?= Date: Thu, 22 Jan 2026 15:15:12 +0800 Subject: [PATCH] support set/remove table properties --- .../org/apache/fluss/spark/SparkCatalog.scala | 23 ++++++++++++++- .../apache/fluss/spark/SparkConversions.scala | 18 ++++++++++-- .../apache/fluss/spark/SparkCatalogTest.scala | 28 +++++++++++++++++++ 3 files changed, 65 insertions(+), 4 deletions(-) diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkCatalog.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkCatalog.scala index 842ef9b395..1a80b234b5 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkCatalog.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkCatalog.scala @@ -80,7 +80,28 @@ class SparkCatalog extends TableCatalog with SupportsFlussNamespaces with WithFl } override def alterTable(ident: Identifier, changes: TableChange*): Table = { - throw new UnsupportedOperationException("Altering table is not supported") + if ( + !changes.forall( + e => e.isInstanceOf[TableChange.SetProperty] || e.isInstanceOf[TableChange.RemoveProperty]) + ) { + throw new IllegalArgumentException( + "Altering table only supports set or remove properties for now") + } + try { + admin + .alterTable(toTablePath(ident), SparkConversions.toFlussTableChanges(changes).asJava, false) + .get() + loadTable(ident) + } catch { + case e: ExecutionException => + if (e.getCause.isInstanceOf[TableNotExistException]) { + throw new NoSuchTableException(ident) + } else { + throw e + } + case e: UnsupportedOperationException => + throw new IllegalArgumentException(e) + } } override def dropTable(ident: Identifier): Boolean = { diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkConversions.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkConversions.scala index f85e33f81e..eeb726678d 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkConversions.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkConversions.scala @@ -25,6 +25,7 @@ import org.apache.fluss.types.RowType import org.apache.spark.sql.FlussIdentityTransform import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap +import org.apache.spark.sql.connector.catalog.TableChange import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.types.StructType @@ -54,7 +55,7 @@ object SparkConversions { tableDescriptorBuilder.partitionedBy(partitionKey: _*) val primaryKeys = if (caseInsensitiveProps.contains(PRIMARY_KEY.key)) { - val pks = caseInsensitiveProps.get(PRIMARY_KEY.key).get.split(",") + val pks = caseInsensitiveProps.get(PRIMARY_KEY.key).get.split(",").map(_.trim) schemaBuilder.primaryKey(pks: _*) pks } else { @@ -64,7 +65,7 @@ object SparkConversions { if (caseInsensitiveProps.contains(BUCKET_NUMBER.key)) { val bucketNum = caseInsensitiveProps.get(BUCKET_NUMBER.key).get.toInt val bucketKeys = if (caseInsensitiveProps.contains(BUCKET_KEY.key)) { - caseInsensitiveProps.get(BUCKET_KEY.key).get.split(",") + caseInsensitiveProps.get(BUCKET_KEY.key).get.split(",").map(_.trim) } else { primaryKeys.filterNot(partitionKey.contains) } @@ -76,7 +77,7 @@ object SparkConversions { } val (tableProps, customProps) = - caseInsensitiveProps.filterNot(SPARK_TABLE_OPTIONS.contains).partition { + caseInsensitiveProps.filterNot(e => SPARK_TABLE_OPTIONS.contains(e._1)).partition { case (key, _) => key.startsWith(FlussConfigUtils.TABLE_PREFIX) } @@ -97,4 +98,15 @@ object SparkConversions { } partitionKeys.toArray } + + def toFlussTableChanges(changes: Seq[TableChange]): Seq[org.apache.fluss.metadata.TableChange] = { + changes.map { + case p: TableChange.SetProperty => + org.apache.fluss.metadata.TableChange.set(p.property(), p.value()) + case p: TableChange.RemoveProperty => + org.apache.fluss.metadata.TableChange.reset(p.property()) + // TODO Add full support for table changes + case _ => throw new UnsupportedOperationException("Unsupported table change") + } + } } diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkCatalogTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkCatalogTest.scala index 4c4bec355a..1805f3fc5c 100644 --- a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkCatalogTest.scala +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkCatalogTest.scala @@ -17,6 +17,7 @@ package org.apache.fluss.spark +import org.apache.fluss.config.ConfigOptions import org.apache.fluss.metadata._ import org.apache.fluss.types.{DataTypes, RowType} @@ -192,6 +193,33 @@ class SparkCatalogTest extends FlussSparkTestBase { checkAnswer(sql("SHOW DATABASES"), Row(DEFAULT_DATABASE) :: Nil) } + test("Catalog: set/remove table properties") { + withTable("t") { + sql( + s"CREATE TABLE $DEFAULT_DATABASE.t (id int, name string) TBLPROPERTIES('key1' = 'value1', '${SparkConnectorOptions.BUCKET_NUMBER.key()}' = 3)") + var flussTable = admin.getTableInfo(createTablePath("t")).get() + assertResult( + Map(ConfigOptions.TABLE_REPLICATION_FACTOR.key() -> "1"), + "check table properties")(flussTable.getProperties.toMap.asScala) + assert( + flussTable.getCustomProperties.toMap.asScala.getOrElse("key1", "non-exists") == "value1") + + sql(s"ALTER TABLE t SET TBLPROPERTIES('key1' = 'value2')") + flussTable = admin.getTableInfo(createTablePath("t")).get() + assertResult( + Map(ConfigOptions.TABLE_REPLICATION_FACTOR.key() -> "1"), + "check table properties")(flussTable.getProperties.toMap.asScala) + assert( + flussTable.getCustomProperties.toMap.asScala.getOrElse("key1", "non-exists") == "value2") + + sql(s"ALTER TABLE t UNSET TBLPROPERTIES('key1')") + flussTable = admin.getTableInfo(createTablePath("t")).get() + assert(!flussTable.getCustomProperties.toMap.asScala.contains("key1")) + + sql(s"ALTER TABLE t UNSET TBLPROPERTIES('key1')") + } + } + test("Partition: show partitions") { withTable("t") { sql(s"CREATE TABLE t (id int, name string, pt1 string, pt2 int) PARTITIONED BY (pt1, pt2)")