Skip to content

Commit 94ebf66

Browse files
luoyuxialuoyuxia
committed
refine doc
1 parent b0a754a commit 94ebf66

File tree

7 files changed

+52
-72
lines changed

7 files changed

+52
-72
lines changed

‎fluss-client/src/test/java/org/apache/fluss/client/table/LakeEnableTableITCase.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.apache.fluss.metadata.TablePath;
3333
import org.apache.fluss.types.DataTypes;
3434

35+
import org.junit.jupiter.api.BeforeEach;
3536
import org.junit.jupiter.api.Test;
3637

3738
import java.util.Arrays;
@@ -47,6 +48,16 @@
4748
/** IT case for lake enable table. */
4849
class LakeEnableTableITCase extends ClientToServerITCaseBase {
4950

51+
@BeforeEach
52+
void beforeEach() throws Exception {
53+
admin.alterClusterConfigs(
54+
Arrays.asList(
55+
new AlterConfig(DATALAKE_FORMAT.key(), null, AlterConfigOpType.SET),
56+
new AlterConfig(
57+
DATALAKE_ENABLED.key(), null, AlterConfigOpType.SET)))
58+
.get();
59+
}
60+
5061
@Test
5162
void testCannotEnableDatalakeForTableCreatedBeforeClusterEnabledDatalake() throws Exception {
5263
String databaseName = "test_db";
@@ -230,7 +241,7 @@ void testCannotEnableTableWhenTableFormatDiffersFromClusterFormat() throws Excep
230241
@Test
231242
void testEnableTableAfterClusterEnablesDataLake() throws Exception {
232243
String databaseName = "test_db";
233-
String tableName = "test_table_before_cluster_enable";
244+
String tableName = "test_enable_datalake_table";
234245
TablePath tablePath = TablePath.of(databaseName, tableName);
235246

236247
admin.createDatabase(databaseName, DatabaseDescriptor.EMPTY, true).get();

‎fluss-server/src/main/java/org/apache/fluss/server/coordinator/LakeCatalogDynamicLoader.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,11 +75,13 @@ public void validate(Configuration newConfig) throws ConfigException {
7575
// set, rather than only the merged result. We accept this for now because
7676
// table-level enablement is still validated, and enabling datalake for a table
7777
// will fail if datalake.format is not configured.
78-
boolean explicitDataLakeEnabled = newConfig.getOptional(DATALAKE_ENABLED).orElse(false);
79-
if (explicitDataLakeEnabled && newDatalakeFormat == null) {
78+
Optional<Boolean> optDataLakeEnabled = newConfig.getOptional(DATALAKE_ENABLED);
79+
if (optDataLakeEnabled.isPresent()
80+
&& optDataLakeEnabled.get()
81+
&& newDatalakeFormat == null) {
8082
throw new ConfigException(
8183
String.format(
82-
"'%s' must be configured when '%s' is explicitly set.",
84+
"'%s' must be configured when '%s' is explicitly set to true.",
8385
DATALAKE_FORMAT.key(), DATALAKE_ENABLED.key()));
8486
}
8587

‎fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -129,11 +129,17 @@ public static void validateTableDescriptor(
129129

130130
private static void checkTableLakeFormatMatchesCluster(
131131
Configuration tableConf, @Nullable DataLakeFormat clusterDataLakeFormat) {
132+
if (clusterDataLakeFormat == null) {
133+
return;
134+
}
135+
136+
if (!tableConf.get(ConfigOptions.TABLE_DATALAKE_ENABLED)) {
137+
return;
138+
}
139+
132140
Optional<DataLakeFormat> tableDataLakeFormat =
133141
tableConf.getOptional(ConfigOptions.TABLE_DATALAKE_FORMAT);
134-
if (clusterDataLakeFormat != null
135-
&& tableDataLakeFormat.isPresent()
136-
&& tableDataLakeFormat.get() != clusterDataLakeFormat) {
142+
if (tableDataLakeFormat.isPresent() && tableDataLakeFormat.get() != clusterDataLakeFormat) {
137143
throw new InvalidConfigException(
138144
String.format(
139145
"'%s' ('%s') must match cluster '%s' ('%s') when '%s' is enabled.",

‎fluss-server/src/test/java/org/apache/fluss/server/DynamicConfigChangeTest.java‎

Lines changed: 3 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -531,7 +531,7 @@ public void reconfigure(Configuration newConfig) {
531531
}
532532

533533
@Test
534-
void testExplicitDatalakeEnabledRequiresFormat() throws Exception {
534+
void testExplicitDataLakeEnabledRequiresDataLakeFormat() throws Exception {
535535
try (LakeCatalogDynamicLoader lakeCatalogDynamicLoader =
536536
new LakeCatalogDynamicLoader(new Configuration(), null, true)) {
537537
DynamicConfigManager dynamicConfigManager =
@@ -545,48 +545,11 @@ void testExplicitDatalakeEnabledRequiresFormat() throws Exception {
545545
Collections.singletonList(
546546
new AlterConfig(
547547
DATALAKE_ENABLED.key(),
548-
"false",
548+
"true",
549549
AlterConfigOpType.SET))))
550550
.isInstanceOf(ConfigException.class)
551551
.hasMessageContaining(
552-
"'datalake.format' must be configured when 'datalake.enabled' is explicitly set.");
553-
}
554-
}
555-
556-
@Test
557-
void testPreBindOnlyModeDoesNotCreateLakeCatalog() throws Exception {
558-
try (LakeCatalogDynamicLoader lakeCatalogDynamicLoader =
559-
new LakeCatalogDynamicLoader(new Configuration(), null, true)) {
560-
DynamicConfigManager dynamicConfigManager =
561-
new DynamicConfigManager(zookeeperClient, new Configuration(), true);
562-
dynamicConfigManager.register(lakeCatalogDynamicLoader);
563-
dynamicConfigManager.startup();
564-
565-
dynamicConfigManager.alterConfigs(
566-
Arrays.asList(
567-
new AlterConfig(DATALAKE_FORMAT.key(), "paimon", AlterConfigOpType.SET),
568-
new AlterConfig(DATALAKE_ENABLED.key(), "false", AlterConfigOpType.SET),
569-
new AlterConfig(
570-
"datalake.paimon.metastore",
571-
"filesystem",
572-
AlterConfigOpType.SET)));
573-
574-
assertThat(lakeCatalogDynamicLoader.getLakeCatalogContainer().getDataLakeFormat())
575-
.isEqualTo(PAIMON);
576-
assertThat(
577-
lakeCatalogDynamicLoader
578-
.getLakeCatalogContainer()
579-
.isClusterDataLakeTableEnabled())
580-
.isFalse();
581-
assertThat(lakeCatalogDynamicLoader.getLakeCatalogContainer().getLakeCatalog())
582-
.isNull();
583-
assertThat(
584-
lakeCatalogDynamicLoader
585-
.getLakeCatalogContainer()
586-
.getDefaultTableLakeOptions())
587-
.isEqualTo(
588-
Collections.singletonMap(
589-
"table.datalake.paimon.metastore", "filesystem"));
552+
"'datalake.format' must be configured when 'datalake.enabled' is explicitly set to true.");
590553
}
591554
}
592555
}

‎website/docs/maintenance/configuration.md‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -188,9 +188,10 @@ More metrics example could be found in [Observability - Metric Reporters](observ
188188
| metrics.reporter.prometheus-push.grouping-key | String | (None) | Specifies the grouping key which is the group and global labels of all metrics. The label name and value are separated by '=', and labels are separated by ';', e.g., k1=v1;k2=v2. |
189189
## Lakehouse
190190

191-
| Option | Type | Default | Description |
192-
|-----------------|------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
193-
| datalake.format | Enum | (None) | The datalake format used by of Fluss to be as lakehouse storage. Currently, supported formats are Paimon, Iceberg, and Lance. In the future, more kinds of data lake format will be supported, such as DeltaLake or Hudi. |
191+
| Option | Type | Default | Description |
192+
|------------------|---------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
193+
| datalake.enabled | Boolean | (None) | Whether the Fluss cluster is ready to create and manage lakehouse tables. If unset, Fluss keeps the legacy behavior where configuring `datalake.format` also enables lakehouse tables. If set to `false`, Fluss pre-binds the lake format for newly created tables but does not allow lakehouse tables yet. If set to `true`, Fluss fully enables lakehouse tables. When this option is explicitly configured to true, `datalake.format` must also be configured. |
194+
| datalake.format | Enum | (None) | The datalake format used by of Fluss to be as lakehouse storage. Currently, supported formats are Paimon, Iceberg, and Lance. In the future, more kinds of data lake format will be supported, such as DeltaLake or Hudi. |
194195

195196
## Kafka
196197

‎website/docs/maintenance/operations/updating-configs.md‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ Fluss allows you to update cluster or table configurations dynamically without r
1313
From Fluss version 0.8 onwards, some of the server configs can be updated without restarting the server.
1414

1515
Currently, the supported dynamically updatable server configurations include:
16-
- `datalake.enabled`: Control whether the cluster is ready to create and manage lakehouse tables. When this option is explicitly configured, `datalake.format` must also be configured.
16+
- `datalake.enabled`: Control whether the cluster is ready to create and manage lakehouse tables. When this option is explicitly configured to true, `datalake.format` must also be configured.
1717
- `datalake.format`: Specify the lakehouse format, e.g., `paimon`, `iceberg`. When enabling lakehouse storage explicitly, use it together with `datalake.enabled = true`.
1818
- Options with prefix `datalake.${datalake.format}`
1919
- `kv.rocksdb.shared-rate-limiter.bytes-per-sec`: Control RocksDB flush and compaction write rate shared across all RocksDB instances on the TabletServer. The rate limiter is always enabled. Set to a lower value (e.g., 100MB) to limit the rate, or a very high value to effectively disable rate limiting.
@@ -32,16 +32,10 @@ admin.alterClusterConfigs(
3232
new AlterConfig(DATALAKE_ENABLED.key(), "true", AlterConfigOpType.SET),
3333
new AlterConfig(DATALAKE_FORMAT.key(), "paimon", AlterConfigOpType.SET)));
3434

35-
// Pre-bind the lakehouse format without enabling lakehouse tables
36-
admin.alterClusterConfigs(
37-
Arrays.asList(
38-
new AlterConfig(DATALAKE_ENABLED.key(), "false", AlterConfigOpType.SET),
39-
new AlterConfig(DATALAKE_FORMAT.key(), "paimon", AlterConfigOpType.SET)));
40-
41-
// Return to legacy behavior where configuring datalake.format alone also enables lakehouse tables
35+
// Disable lakehouse storage
4236
admin.alterClusterConfigs(
4337
Collections.singletonList(
44-
new AlterConfig(DATALAKE_ENABLED.key(), null, AlterConfigOpType.DELETE)));
38+
new AlterConfig(DATALAKE_ENABLED.key(), "false", AlterConfigOpType.SET)));
4539

4640
// Set RocksDB shared rate limiter to 200MB/sec
4741
admin.alterClusterConfigs(
@@ -54,8 +48,6 @@ The `AlterConfig` class contains three properties:
5448
* `value`: The configuration value to be set (e.g., `paimon`)
5549
* `opType`: The operation type, either `AlterConfigOpType.SET` or `AlterConfigOpType.DELETE`
5650

57-
If `datalake.enabled` is explicitly configured, `datalake.format` must also be configured in the cluster.
58-
5951
### Using Flink Stored Procedures
6052

6153
For certain configurations, Fluss provides convenient Flink stored procedures that can be called directly from Flink SQL. See [Procedures](engine-flink/procedures.md#cluster-configuration-procedures) for detailed documentation on using `get_cluster_config` and `set_cluster_config` procedures.

‎website/docs/maintenance/operations/upgrade-notes-0.10.md‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,35 +5,40 @@ sidebar_position: 4
55

66
# Upgrade Notes from v0.9 to v0.10
77

8-
## New `datalake.enabled` Cluster Configuration
8+
## Cluster Configuration Changes
99

10-
Starting from v0.10, Fluss introduces the cluster-level configuration `datalake.enabled` to control whether the cluster is ready to create and manage lakehouse tables.
10+
### New `datalake.enabled` Cluster Configuration
1111

12-
### Behavior Changes
12+
Starting in v0.10, Fluss introduces the cluster-level configuration `datalake.enabled` to control whether the cluster is ready to create and manage lakehouse tables.
1313

14-
- If `datalake.enabled` is unset, Fluss keeps the legacy behavior: configuring `datalake.format` alone also enables lakehouse tables.
15-
- If `datalake.enabled = false`, Fluss only pre-binds the lake format for newly created tables and does not allow lakehouse tables yet.
14+
#### Behavior Changes
15+
16+
- If `datalake.enabled` is unset, Fluss preserves the legacy behavior: configuring `datalake.format` alone also enables lakehouse tables.
17+
- If `datalake.enabled = false`, Fluss pre-binds the lake format for newly created tables but does not allow lakehouse tables yet.
1618
- If `datalake.enabled = true`, Fluss fully enables lakehouse tables.
1719
- If `datalake.enabled` is explicitly configured, `datalake.format` must also be configured.
1820

19-
### Recommended Configuration
21+
#### Recommended Configuration
2022

21-
If you want to enable lakehouse tables on the cluster, configure both options together:
23+
To enable lakehouse tables for the cluster, configure both options together:
2224

2325
```yaml
2426
datalake.enabled: true
2527
datalake.format: paimon
2628
```
2729
28-
If you only want to pre-bind the lake format without enabling lakehouse tables yet, configure:
30+
To pre-bind the lake format without enabling lakehouse tables yet, configure:
2931
3032
```yaml
3133
datalake.enabled: false
3234
datalake.format: paimon
3335
```
3436
35-
### Documentation Updates for Existing Deployments
37+
This mode is useful when you want newly created tables to carry the lake format in advance, while postponing lakehouse enablement at the cluster level.
38+
After `datalake.enabled` is later set to `true`, tables created under this configuration can still turn on `table.datalake.enabled` without being recreated.
39+
40+
#### Notes for Existing Deployments
3641

37-
If your existing deployment or internal scripts only set `datalake.format`, they will continue to work with the legacy behavior as long as `datalake.enabled` is left unset.
42+
If your existing deployment or internal scripts only set `datalake.format`, they will continue to work with the legacy behavior as long as `datalake.enabled` remains unset.
3843

39-
However, for new configuration examples and operational guidance, prefer configuring `datalake.enabled` explicitly together with `datalake.format`.
44+
For new configuration examples and operational guidance, we recommend explicitly configuring `datalake.enabled` together with `datalake.format`.

0 commit comments

Comments
 (0)