Skip to content

Commit 07d6a30

Browse files
committed
[AMORO-4099] Add table-summary metric collection option for non-optimizing tables
- Allow collecting table_summary metrics when self-optimizing is disabled by setting table-summary.enabled=true - Fix periodic collection bug: remove optimizingNotNecessary() call in summary-only branch to prevent snapshot gate from blocking subsequent collections - Separate property key from self-optimizing prefix: self-optimizing.table-summary.enabled -> table-summary.enabled - Add debug logging for table summary collection path - Add comprehensive test coverage for summary-only mode Signed-off-by: Jiwon Park <jpark92@outlook.kr>
1 parent 743e6f1 commit 07d6a30

5 files changed

Lines changed: 473 additions & 8 deletions

File tree

amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/TableRuntimeRefreshExecutor.java

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -61,11 +61,10 @@ protected long getNextExecutingTime(TableRuntime tableRuntime) {
6161
}
6262

6363
private void tryEvaluatingPendingInput(DefaultTableRuntime tableRuntime, MixedTable table) {
64-
// only evaluate pending input when optimizing is enabled and in idle state
6564
OptimizingConfig optimizingConfig = tableRuntime.getOptimizingConfig();
6665
if (optimizingConfig.isEnabled()
6766
&& tableRuntime.getOptimizingStatus().equals(OptimizingStatus.IDLE)) {
68-
67+
// Evaluate pending input and collect table summary when optimizing is enabled and idle
6968
if (optimizingConfig.isMetadataBasedTriggerEnabled()
7069
&& !MetadataBasedEvaluationEvent.isEvaluatingNecessary(
7170
optimizingConfig, table, tableRuntime.getLastPlanTime())) {
@@ -89,6 +88,16 @@ private void tryEvaluatingPendingInput(DefaultTableRuntime tableRuntime, MixedTa
8988
tableRuntime.optimizingNotNecessary();
9089
}
9190
tableRuntime.setTableSummary(evaluator.getPendingInput());
91+
} else if (!optimizingConfig.isEnabled() && optimizingConfig.isTableSummaryEnabled()) {
92+
// Collect table summary metrics even when optimizing is disabled
93+
logger.info(
94+
"{} collecting table summary (optimizing disabled, tableSummary enabled)",
95+
tableRuntime.getTableIdentifier());
96+
AbstractOptimizingEvaluator evaluator =
97+
IcebergTableUtil.createOptimizingEvaluator(tableRuntime, table, maxPendingPartitions);
98+
AbstractOptimizingEvaluator.PendingInput summary = evaluator.getPendingInput();
99+
logger.info("{} table summary collected: {}", tableRuntime.getTableIdentifier(), summary);
100+
tableRuntime.setTableSummary(summary);
92101
}
93102
}
94103

@@ -122,12 +131,26 @@ public void execute(TableRuntime tableRuntime) {
122131
AmoroTable<?> table = loadTable(tableRuntime);
123132
defaultTableRuntime.refresh(table);
124133
MixedTable mixedTable = (MixedTable) table.originalTable();
125-
if ((mixedTable.isKeyedTable()
126-
&& (lastOptimizedSnapshotId != defaultTableRuntime.getCurrentSnapshotId()
127-
|| lastOptimizedChangeSnapshotId
128-
!= defaultTableRuntime.getCurrentChangeSnapshotId()))
129-
|| (mixedTable.isUnkeyedTable()
130-
&& lastOptimizedSnapshotId != defaultTableRuntime.getCurrentSnapshotId())) {
134+
boolean snapshotChanged =
135+
(mixedTable.isKeyedTable()
136+
&& (lastOptimizedSnapshotId != defaultTableRuntime.getCurrentSnapshotId()
137+
|| lastOptimizedChangeSnapshotId
138+
!= defaultTableRuntime.getCurrentChangeSnapshotId()))
139+
|| (mixedTable.isUnkeyedTable()
140+
&& lastOptimizedSnapshotId != defaultTableRuntime.getCurrentSnapshotId());
141+
OptimizingConfig optimizingConfig = defaultTableRuntime.getOptimizingConfig();
142+
boolean tableSummaryOnly =
143+
!optimizingConfig.isEnabled() && optimizingConfig.isTableSummaryEnabled();
144+
logger.info(
145+
"{} refresh check: snapshotChanged={}, optimizingEnabled={}, tableSummaryEnabled={}, "
146+
+ "lastOptimizedSnapshotId={}, currentSnapshotId={}",
147+
tableRuntime.getTableIdentifier(),
148+
snapshotChanged,
149+
optimizingConfig.isEnabled(),
150+
optimizingConfig.isTableSummaryEnabled(),
151+
lastOptimizedSnapshotId,
152+
defaultTableRuntime.getCurrentSnapshotId());
153+
if (snapshotChanged || tableSummaryOnly) {
131154
tryEvaluatingPendingInput(defaultTableRuntime, mixedTable);
132155
}
133156
} catch (Throwable throwable) {

amoro-ams/src/main/java/org/apache/amoro/server/table/TableConfigurations.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,11 @@ public static OptimizingConfig parseOptimizingConfig(Map<String, String> propert
232232
properties,
233233
TableProperties.ENABLE_SELF_OPTIMIZING,
234234
TableProperties.ENABLE_SELF_OPTIMIZING_DEFAULT))
235+
.setTableSummaryEnabled(
236+
CompatiblePropertyUtil.propertyAsBoolean(
237+
properties,
238+
TableProperties.TABLE_SUMMARY_ENABLED,
239+
TableProperties.TABLE_SUMMARY_ENABLED_DEFAULT))
235240
.setAllowPartialCommit(
236241
CompatiblePropertyUtil.propertyAsBoolean(
237242
properties,

0 commit comments

Comments
 (0)