");
+ }
+
+ private static WalFileAnalysis analyze(final File walFile) throws IOException {
+ try (RandomAccessFile raf = new RandomAccessFile(walFile, "r");
+ FileChannel channel = raf.getChannel()) {
+ final long totalBytes = channel.size();
+ final String version = detectVersion(channel, totalBytes);
+ final int headMagicBytes = getHeadMagicBytes(version);
+ final int tailMagicBytes = getTailMagicBytes(version);
+
+ final WalFileAnalysis analysis = new WalFileAnalysis(walFile, version, totalBytes);
+ analysis.headMagicBytes = Math.min(totalBytes, headMagicBytes);
+
+ if (totalBytes <= headMagicBytes) {
+ analysis.note = "header-only WAL file (magic only, no body/footer)";
+ return analysis;
+ }
+
+ if (!hasTrailingMagic(channel, totalBytes, version)) {
+ analysis.note = "missing trailing magic/footer, file may be open or broken";
+ return analysis;
+ }
+
+ analysis.tailMagicBytes = tailMagicBytes;
+ analysis.metadataSizeFieldBytes = METADATA_SIZE_FIELD_BYTES;
+
+ final long metadataSizeFieldPos = totalBytes - tailMagicBytes - METADATA_SIZE_FIELD_BYTES;
+ if (metadataSizeFieldPos < headMagicBytes) {
+ analysis.note = "invalid metadata size position";
+ return analysis;
+ }
+
+ final int metadataBytes = readInt(channel, metadataSizeFieldPos);
+ analysis.metadataBytes = metadataBytes;
+ analysis.footerStartOffset = metadataSizeFieldPos - metadataBytes;
+ if (analysis.footerStartOffset < headMagicBytes) {
+ analysis.note = "invalid footer start offset";
+ return analysis;
+ }
+
+ final long markerOffset = analysis.footerStartOffset - WAL_FILE_INFO_END_MARKER_BYTES;
+ if (markerOffset < headMagicBytes) {
+ analysis.note = "invalid end-marker offset";
+ return analysis;
+ }
+
+ analysis.endMarkerBytes = WAL_FILE_INFO_END_MARKER_BYTES;
+ analysis.segmentStartOffset = headMagicBytes;
+ analysis.segmentEndOffsetExclusive = markerOffset;
+ analysis.segmentRegionBytes = Math.max(0L, markerOffset - headMagicBytes);
+
+ scanSegments(channel, analysis);
+ parseFooter(channel, analysis);
+ return analysis;
+ }
+ }
+
+ private static void scanSegments(final FileChannel channel, final WalFileAnalysis analysis)
+ throws IOException {
+ long offset = analysis.segmentStartOffset;
+ while (offset < analysis.segmentEndOffsetExclusive) {
+ if (analysis.segmentEndOffsetExclusive - offset < SEGMENT_HEADER_BASE_BYTES) {
+ analysis.segmentParseWarning =
+ "remaining bytes are smaller than a segment header at offset " + offset;
+ return;
+ }
+
+ final ByteBuffer headerBuffer = ByteBuffer.allocate(SEGMENT_HEADER_BASE_BYTES);
+ readFully(channel, headerBuffer, offset);
+ headerBuffer.flip();
+
+ final CompressionType compressionType = CompressionType.deserialize(headerBuffer.get());
+ final int dataInDiskBytes = headerBuffer.getInt();
+ int headerBytes = SEGMENT_HEADER_BASE_BYTES;
+ if (compressionType != CompressionType.UNCOMPRESSED) {
+ headerBytes += COMPRESSED_SEGMENT_EXTRA_HEADER_BYTES;
+ }
+
+ final long nextOffset = offset + headerBytes + dataInDiskBytes;
+ if (nextOffset > analysis.segmentEndOffsetExclusive) {
+ analysis.segmentParseWarning =
+ String.format(
+ Locale.ROOT,
+ "segment at offset %d exceeds body boundary (%d > %d)",
+ offset,
+ nextOffset,
+ analysis.segmentEndOffsetExclusive);
+ return;
+ }
+
+ analysis.segmentCount++;
+ analysis.segmentHeaderBytes += headerBytes;
+ analysis.segmentPayloadBytes += dataInDiskBytes;
+ if (compressionType != CompressionType.UNCOMPRESSED) {
+ analysis.compressedSegmentCount++;
+ }
+ offset = nextOffset;
+ }
+
+ if (offset != analysis.segmentEndOffsetExclusive) {
+ analysis.segmentParseWarning =
+ String.format(
+ Locale.ROOT,
+ "segment parser stopped at %d but expected %d",
+ offset,
+ analysis.segmentEndOffsetExclusive);
+ }
+ }
+
+ private static void parseFooter(final FileChannel channel, final WalFileAnalysis analysis)
+ throws IOException {
+ if (analysis.metadataBytes <= 0) {
+ return;
+ }
+
+ final ByteBuffer metadataBuffer = ByteBuffer.allocate(analysis.metadataBytes);
+ readFully(channel, metadataBuffer, analysis.footerStartOffset);
+ metadataBuffer.flip();
+
+ if (metadataBuffer.remaining() < Long.BYTES + Integer.BYTES) {
+ analysis.footerWarning = "metadata buffer is too small";
+ return;
+ }
+
+ metadataBuffer.getLong();
+ analysis.firstSearchIndexBytes = Long.BYTES;
+ final int entryCount = metadataBuffer.getInt();
+ analysis.entryCount = entryCount;
+ analysis.entryCountBytes = Integer.BYTES;
+
+ analysis.bufferSizeArrayBytes = (long) entryCount * Integer.BYTES;
+ for (int i = 0; i < entryCount; i++) {
+ metadataBuffer.getInt();
+ }
+
+ final boolean serializedEmptyV3WithoutMemTableCount =
+ V3_MAGIC.equals(analysis.version)
+ && entryCount == 0
+ && metadataBuffer.remaining() == V3_EMPTY_METADATA_REMAINING_WITHOUT_MEMTABLE_COUNT;
+
+ if (metadataBuffer.hasRemaining() && !serializedEmptyV3WithoutMemTableCount) {
+ analysis.memTableCountFieldBytes = Integer.BYTES;
+ analysis.memTableCount = metadataBuffer.getInt();
+ analysis.memTableIdsBytes = (long) analysis.memTableCount * Long.BYTES;
+ for (int i = 0; i < analysis.memTableCount; i++) {
+ metadataBuffer.getLong();
+ }
+ }
+
+ if (V3_MAGIC.equals(analysis.version) && metadataBuffer.hasRemaining()) {
+ if (metadataBuffer.remaining() < Long.BYTES * 2) {
+ analysis.footerWarning = "V3 metadata is truncated before min/max timestamp range";
+ return;
+ }
+
+ analysis.minMaxDataTsBytes = Long.BYTES * 2L;
+ metadataBuffer.getLong();
+ metadataBuffer.getLong();
+
+ final long requiredWriterMetadataBytes =
+ (long) entryCount * Long.BYTES * 2 + Short.BYTES * 2 + Integer.BYTES;
+ if (metadataBuffer.remaining() < requiredWriterMetadataBytes) {
+ analysis.footerWarning = "V3 metadata is truncated before writer progress arrays";
+ return;
+ }
+
+ analysis.physicalTimesBytes = (long) entryCount * Long.BYTES;
+ analysis.localSeqsBytes = (long) entryCount * Long.BYTES;
+ for (int i = 0; i < entryCount; i++) {
+ metadataBuffer.getLong();
+ }
+ for (int i = 0; i < entryCount; i++) {
+ metadataBuffer.getLong();
+ }
+
+ analysis.defaultWriterIdentityBytes = Short.BYTES * 2L;
+ metadataBuffer.getShort();
+ metadataBuffer.getShort();
+
+ analysis.overrideCountFieldBytes = Integer.BYTES;
+ analysis.overrideCount = metadataBuffer.getInt();
+
+ analysis.overrideIndexesBytes = (long) analysis.overrideCount * Integer.BYTES;
+ analysis.overrideNodeIdsBytes = (long) analysis.overrideCount * Short.BYTES;
+ analysis.overrideWriterEpochsBytes = (long) analysis.overrideCount * Short.BYTES;
+
+ for (int i = 0; i < analysis.overrideCount; i++) {
+ metadataBuffer.getInt();
+ }
+ for (int i = 0; i < analysis.overrideCount; i++) {
+ metadataBuffer.getShort();
+ }
+ for (int i = 0; i < analysis.overrideCount; i++) {
+ metadataBuffer.getShort();
+ }
+ }
+
+ analysis.unknownMetadataBytes = metadataBuffer.remaining();
+ }
+
+ private static String detectVersion(final FileChannel channel, final long totalBytes)
+ throws IOException {
+ if (totalBytes >= V3_MAGIC.length()
+ && readString(channel, 0, V3_MAGIC.length()).equals(V3_MAGIC)) {
+ return V3_MAGIC;
+ }
+ if (totalBytes >= V2_MAGIC.length()
+ && readString(channel, 0, V2_MAGIC.length()).equals(V2_MAGIC)) {
+ return V2_MAGIC;
+ }
+ if (totalBytes >= V1_MAGIC.length()
+ && readString(channel, totalBytes - V1_MAGIC.length(), V1_MAGIC.length())
+ .equals(V1_MAGIC)) {
+ return V1_MAGIC;
+ }
+ return "UNKNOWN";
+ }
+
+ private static int getHeadMagicBytes(final String version) {
+ if (V3_MAGIC.equals(version)) {
+ return V3_MAGIC.length();
+ }
+ if (V2_MAGIC.equals(version)) {
+ return V2_MAGIC.length();
+ }
+ return 0;
+ }
+
+ private static int getTailMagicBytes(final String version) {
+ if (V3_MAGIC.equals(version)) {
+ return V3_MAGIC.length();
+ }
+ if (V2_MAGIC.equals(version)) {
+ return V2_MAGIC.length();
+ }
+ if (V1_MAGIC.equals(version)) {
+ return V1_MAGIC.length();
+ }
+ return 0;
+ }
+
+ private static boolean hasTrailingMagic(
+ final FileChannel channel, final long totalBytes, final String version) throws IOException {
+ final int tailMagicBytes = getTailMagicBytes(version);
+ if (tailMagicBytes <= 0 || totalBytes < tailMagicBytes) {
+ return false;
+ }
+ return readString(channel, totalBytes - tailMagicBytes, tailMagicBytes).equals(version);
+ }
+
+ private static String readString(final FileChannel channel, final long offset, final int length)
+ throws IOException {
+ final ByteBuffer buffer = ByteBuffer.allocate(length);
+ readFully(channel, buffer, offset);
+ buffer.flip();
+ return StandardCharsets.UTF_8.decode(buffer).toString();
+ }
+
+ private static int readInt(final FileChannel channel, final long offset) throws IOException {
+ final ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
+ readFully(channel, buffer, offset);
+ buffer.flip();
+ return buffer.getInt();
+ }
+
+ private static void readFully(
+ final FileChannel channel, final ByteBuffer buffer, final long offset) throws IOException {
+ long position = offset;
+ while (buffer.hasRemaining()) {
+ final int bytesRead = channel.read(buffer, position);
+ if (bytesRead < 0) {
+ throw new IOException("Unexpected EOF while reading at offset " + position);
+ }
+ position += bytesRead;
+ }
+ }
+
+ private static void printAnalysis(final WalFileAnalysis analysis) {
+ System.out.println("=== WAL File Layout Analysis ===");
+ System.out.println("file: " + analysis.file.getAbsolutePath());
+ System.out.println("version: " + analysis.version);
+ System.out.println("total: " + formatBytes(analysis.totalBytes));
+ if (analysis.note != null) {
+ System.out.println("note: " + analysis.note);
+ }
+ System.out.println();
+
+ printSection("head magic", analysis.headMagicBytes, analysis.totalBytes);
+ printSection("segment headers", analysis.segmentHeaderBytes, analysis.totalBytes);
+ printSection("segment payload", analysis.segmentPayloadBytes, analysis.totalBytes);
+ printSection("wal end marker", analysis.endMarkerBytes, analysis.totalBytes);
+ printSection("footer metadata", analysis.metadataBytes, analysis.totalBytes);
+ printSection("metadata size field", analysis.metadataSizeFieldBytes, analysis.totalBytes);
+ printSection("tail magic", analysis.tailMagicBytes, analysis.totalBytes);
+ final long accountedBytes =
+ analysis.headMagicBytes
+ + analysis.segmentHeaderBytes
+ + analysis.segmentPayloadBytes
+ + analysis.endMarkerBytes
+ + analysis.metadataBytes
+ + analysis.metadataSizeFieldBytes
+ + analysis.tailMagicBytes;
+ if (analysis.totalBytes >= accountedBytes) {
+ printSection("unaccounted", analysis.totalBytes - accountedBytes, analysis.totalBytes);
+ }
+
+ System.out.println();
+ System.out.println(
+ String.format(
+ Locale.ROOT,
+ "segments: total=%d, compressed=%d",
+ analysis.segmentCount,
+ analysis.compressedSegmentCount));
+ if (analysis.segmentParseWarning != null) {
+ System.out.println("segment warning: " + analysis.segmentParseWarning);
+ }
+
+ if (analysis.metadataBytes <= 0) {
+ return;
+ }
+
+ System.out.println();
+ System.out.println("=== Footer Breakdown ===");
+ printSection("v2-compatible base", analysis.getV2BaseMetadataBytes(), analysis.totalBytes);
+ if (V3_MAGIC.equals(analysis.version)) {
+ printSection("v3 extension total", analysis.getV3ExtensionBytes(), analysis.totalBytes);
+ System.out.println(
+ String.format(
+ Locale.ROOT,
+ "v3 extension share of footer: %s",
+ formatPercent(analysis.getV3ExtensionBytes(), analysis.metadataBytes)));
+ printSection(" min/max data ts", analysis.minMaxDataTsBytes, analysis.totalBytes);
+ printSection(" physicalTimes[]", analysis.physicalTimesBytes, analysis.totalBytes);
+ printSection(" localSeqs[]", analysis.localSeqsBytes, analysis.totalBytes);
+ printSection(
+ " default writer identity + override count",
+ analysis.defaultWriterIdentityBytes + analysis.overrideCountFieldBytes,
+ analysis.totalBytes);
+ printSection(" overrideIndexes[]", analysis.overrideIndexesBytes, analysis.totalBytes);
+ printSection(" overrideNodeIds[]", analysis.overrideNodeIdsBytes, analysis.totalBytes);
+ printSection(
+ " overrideWriterEpochs[]", analysis.overrideWriterEpochsBytes, analysis.totalBytes);
+ }
+ if (analysis.unknownMetadataBytes > 0) {
+ printSection("unknown metadata tail", analysis.unknownMetadataBytes, analysis.totalBytes);
+ }
+ System.out.println(
+ String.format(
+ Locale.ROOT,
+ "entries=%d, memTables=%d, overrides=%d",
+ analysis.entryCount,
+ analysis.memTableCount,
+ analysis.overrideCount));
+ if (analysis.footerWarning != null) {
+ System.out.println("footer warning: " + analysis.footerWarning);
+ }
+ }
+
+ private static void printSection(final String name, final long bytes, final long totalBytes) {
+ System.out.println(
+ String.format(
+ Locale.ROOT,
+ "%-42s %12s %8s",
+ name + ":",
+ formatBytes(bytes),
+ formatPercent(bytes, totalBytes)));
+ }
+
+ private static String formatBytes(final long bytes) {
+ final long absBytes = Math.abs(bytes);
+ if (absBytes < 1024L) {
+ return bytes + " B";
+ }
+ if (absBytes < 1024L * 1024L) {
+ return String.format(Locale.ROOT, "%.2f KiB", bytes / 1024.0d);
+ }
+ if (absBytes < 1024L * 1024L * 1024L) {
+ return String.format(Locale.ROOT, "%.2f MiB", bytes / 1024.0d / 1024.0d);
+ }
+ return String.format(Locale.ROOT, "%.2f GiB", bytes / 1024.0d / 1024.0d / 1024.0d);
+ }
+
+ private static String formatPercent(final long bytes, final long totalBytes) {
+ if (totalBytes <= 0) {
+ return "N/A";
+ }
+ return String.format(Locale.ROOT, "%.2f%%", bytes * 100.0d / totalBytes);
+ }
+
+ private static final class WalFileAnalysis {
+ private final File file;
+ private final String version;
+ private final long totalBytes;
+
+ private long headMagicBytes;
+ private long segmentHeaderBytes;
+ private long segmentPayloadBytes;
+ private long endMarkerBytes;
+ private int metadataBytes;
+ private long metadataSizeFieldBytes;
+ private long tailMagicBytes;
+
+ private long footerStartOffset;
+ private long segmentStartOffset;
+ private long segmentEndOffsetExclusive;
+ private long segmentRegionBytes;
+
+ private int segmentCount;
+ private int compressedSegmentCount;
+
+ private int entryCount;
+ private int memTableCount;
+ private int overrideCount;
+ private long firstSearchIndexBytes;
+ private long entryCountBytes;
+ private long bufferSizeArrayBytes;
+ private long memTableCountFieldBytes;
+ private long memTableIdsBytes;
+ private long minMaxDataTsBytes;
+ private long physicalTimesBytes;
+ private long localSeqsBytes;
+ private long defaultWriterIdentityBytes;
+ private long overrideCountFieldBytes;
+ private long overrideIndexesBytes;
+ private long overrideNodeIdsBytes;
+ private long overrideWriterEpochsBytes;
+ private long unknownMetadataBytes;
+
+ private String note;
+ private String segmentParseWarning;
+ private String footerWarning;
+
+ private WalFileAnalysis(final File file, final String version, final long totalBytes) {
+ this.file = file;
+ this.version = version;
+ this.totalBytes = totalBytes;
+ }
+
+ private long getV2BaseMetadataBytes() {
+ return firstSearchIndexBytes
+ + entryCountBytes
+ + bufferSizeArrayBytes
+ + memTableCountFieldBytes
+ + memTableIdsBytes;
+ }
+
+ private long getV3ExtensionBytes() {
+ return minMaxDataTsBytes
+ + physicalTimesBytes
+ + localSeqsBytes
+ + defaultWriterIdentityBytes
+ + overrideCountFieldBytes
+ + overrideIndexesBytes
+ + overrideNodeIdsBytes
+ + overrideWriterEpochsBytes;
+ }
+ }
+}
diff --git a/example/subscription/src/main/java/org/apache/iotdb/ConsensusSubscriptionPerfTest.java b/example/subscription/src/main/java/org/apache/iotdb/ConsensusSubscriptionPerfTest.java
new file mode 100644
index 0000000000000..a6fa862da011a
--- /dev/null
+++ b/example/subscription/src/main/java/org/apache/iotdb/ConsensusSubscriptionPerfTest.java
@@ -0,0 +1,1526 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb;
+
+import org.apache.iotdb.rpc.subscription.config.TopicConfig;
+import org.apache.iotdb.rpc.subscription.config.TopicConstant;
+import org.apache.iotdb.rpc.subscription.payload.poll.TopicProgress;
+import org.apache.iotdb.session.subscription.ISubscriptionTreeSession;
+import org.apache.iotdb.session.subscription.SubscriptionTreeSessionBuilder;
+import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer;
+import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumerBuilder;
+import org.apache.iotdb.session.subscription.payload.PollResult;
+import org.apache.iotdb.session.subscription.payload.SubscriptionMessage;
+import org.apache.iotdb.session.subscription.payload.SubscriptionMessageType;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.utils.BitMap;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Objects;
+import java.util.Properties;
+import java.util.Random;
+import java.util.TreeMap;
+import java.util.concurrent.locks.LockSupport;
+
+/**
+ * Manual performance test for consensus subscription.
+ *
+ * Typical usage:
+ *
+ *
+ * java ... org.apache.iotdb.ConsensusSubscriptionPerfTest
+ * java ... org.apache.iotdb.ConsensusSubscriptionPerfTest --topic=topic_perf --group=cg_perf
+ * java ... org.apache.iotdb.ConsensusSubscriptionPerfTest --path=root.db_bench.**
+ * java ... org.apache.iotdb.ConsensusSubscriptionPerfTest --orderMode=per-writer
+ * java ... org.apache.iotdb.ConsensusSubscriptionPerfTest --topic=topic_perf --createTopicOnly=true
+ *
+ *
+ * This tool is designed to be started before a benchmark writer (for example iot-benchmark). It
+ * creates a live topic by default and continuously prints subscription throughput statistics.
+ */
+public class ConsensusSubscriptionPerfTest {
+
+ private static final DateTimeFormatter TIME_FORMATTER =
+ DateTimeFormatter.ofPattern("HH:mm:ss").withZone(ZoneId.systemDefault());
+ private static final long RANDOM_SEEK_CHECKPOINT_INTERVAL_ROWS = 100_000L;
+
+ public static void main(final String[] args) throws Exception {
+ final PerfConfig config = PerfConfig.parse(args);
+
+ if (config.help) {
+ printUsage();
+ return;
+ }
+
+ System.out.println("=== Consensus Subscription Performance Test ===");
+ System.out.println(config);
+
+ if (config.autoCreateTopic) {
+ createTopicIfNeeded(config);
+ }
+
+ if (config.createTopicOnly) {
+ System.out.println(
+ String.format(
+ Locale.ROOT,
+ "[%s] Topic is ready. Exiting due to createTopicOnly=true",
+ nowString()));
+ return;
+ }
+
+ final PerfStats stats = new PerfStats(config.enableEquivalentRowTracking());
+ final RandomSeekController randomSeekController = new RandomSeekController(config.randomSeek);
+ final ScheduledSeekController scheduledSeekController =
+ new ScheduledSeekController(config.seekCaptureRows > 0 && config.seekTriggerNanos > 0);
+ final ConsumerRestartController consumerRestartController =
+ new ConsumerRestartController(config.consumerStopNanos > 0);
+ final ConsumerPauseController consumerPauseController =
+ new ConsumerPauseController(config.consumerPauseEveryRows);
+ long startNanoTime;
+ long lastReportNanoTime;
+ final Snapshot[] lastSnapshot = new Snapshot[1];
+ final ProcessingRateLimiter processingRateLimiter =
+ new ProcessingRateLimiter(config.targetPointsPerSec);
+ SubscriptionTreePullConsumer consumer = null;
+ PollResult lastPollResult = emptyPollResult(stats);
+
+ try {
+ consumer = openAndSubscribeConsumer(config);
+
+ System.out.println(
+ String.format(
+ Locale.ROOT, "[%s] Subscribed. Waiting for benchmark writes...", nowString()));
+
+ if (config.waitBeforePollNanos > 0) {
+ System.out.println(
+ String.format(
+ Locale.ROOT,
+ "[%s] Delaying poll start for %.3f second(s)...",
+ nowString(),
+ config.waitBeforePollSec));
+ LockSupport.parkNanos(config.waitBeforePollNanos);
+ }
+
+ System.out.println(String.format(Locale.ROOT, "[%s] Starting poll loop.", nowString()));
+
+ startNanoTime = System.nanoTime();
+ lastReportNanoTime = startNanoTime;
+ lastSnapshot[0] = Snapshot.capture(stats);
+
+ while (config.durationSec <= 0
+ || nanosToSeconds(System.nanoTime() - startNanoTime) < config.durationSec) {
+ final long loopNowNanoTime = System.nanoTime();
+ final long elapsedNanoTime = loopNowNanoTime - startNanoTime;
+
+ if (shouldStopConsumer(config, consumerRestartController, elapsedNanoTime)
+ && Objects.nonNull(consumer)) {
+ consumerRestartController.stopPerformed = true;
+ consumerRestartController.stoppedNanoTime = System.nanoTime();
+ System.out.println(
+ String.format(
+ Locale.ROOT,
+ "[%s] Consumer polling paused at elapsedSec=%.3f; polling will resume at %.3f second(s).",
+ nowString(),
+ elapsedNanoTime / 1_000_000_000.0d,
+ config.consumerResumeSec));
+ }
+
+ if (shouldPauseConsumerByRows(config, consumerPauseController, stats.totalRows)
+ && Objects.nonNull(consumer)) {
+ consumerPauseController.pausePerformedCount++;
+ consumerPauseController.paused = true;
+ consumerPauseController.stoppedNanoTime = System.nanoTime();
+ consumerPauseController.nextPauseRows = stats.totalRows + config.consumerPauseEveryRows;
+ System.out.println(
+ String.format(
+ Locale.ROOT,
+ "[%s] Consumer paused after rows=%d; polling will resume in %.3f second(s).",
+ nowString(),
+ stats.totalRows,
+ config.consumerPauseDurationSec));
+ }
+
+ if (shouldResumeConsumer(config, consumerRestartController, elapsedNanoTime)
+ && Objects.nonNull(consumer)) {
+ final long resumedNanoTime = System.nanoTime();
+ processingRateLimiter.pauseForDowntime(
+ resumedNanoTime - consumerRestartController.stoppedNanoTime);
+ consumerRestartController.resumePerformed = true;
+ System.out.println(
+ String.format(
+ Locale.ROOT,
+ "[%s] Consumer polling resumed at elapsedSec=%.3f after downtimeSec=%.3f.",
+ nowString(),
+ (resumedNanoTime - startNanoTime) / 1_000_000_000.0d,
+ (resumedNanoTime - consumerRestartController.stoppedNanoTime)
+ / 1_000_000_000.0d));
+ }
+
+ if (shouldResumeConsumerByRows(config, consumerPauseController)
+ && Objects.nonNull(consumer)) {
+ final long resumedNanoTime = System.nanoTime();
+ processingRateLimiter.pauseForDowntime(
+ resumedNanoTime - consumerPauseController.stoppedNanoTime);
+ consumerPauseController.paused = false;
+ System.out.println(
+ String.format(
+ Locale.ROOT,
+ "[%s] Consumer resumed after row-based pause at rows=%d, downtimeSec=%.3f.",
+ nowString(),
+ stats.totalRows,
+ (resumedNanoTime - consumerPauseController.stoppedNanoTime) / 1_000_000_000.0d));
+ }
+
+ final boolean pollingPaused =
+ consumerRestartController.enabled
+ && consumerRestartController.stopPerformed
+ && !consumerRestartController.resumePerformed
+ || consumerPauseController.enabled && consumerPauseController.paused;
+
+ final PollResult pollResult;
+ if (Objects.nonNull(consumer) && !pollingPaused) {
+ pollResult = consumer.pollWithInfo(config.pollTimeoutMs);
+ handlePollResult(
+ pollResult,
+ stats,
+ config.processDelayNanos,
+ processingRateLimiter,
+ config.ingestWallTimeSensor);
+ captureScheduledSeekCheckpoint(consumer, config, stats, scheduledSeekController);
+ captureRandomSeekCheckpoint(consumer, config, stats, randomSeekController);
+ maybePerformScheduledSeek(
+ consumer, config, stats, scheduledSeekController, System.nanoTime() - startNanoTime);
+ maybePerformRandomSeek(consumer, config, stats, randomSeekController);
+ } else {
+ LockSupport.parkNanos(Math.min(100_000_000L, config.pollTimeoutMs * 1_000_000L));
+ pollResult = emptyPollResult(stats);
+ }
+ lastPollResult = pollResult;
+
+ final long nowNanoTime = System.nanoTime();
+ if (nowNanoTime - lastReportNanoTime >= config.reportIntervalSec * 1_000_000_000L) {
+ printReport(
+ "interval",
+ lastSnapshot[0],
+ Snapshot.capture(stats),
+ nowNanoTime - lastReportNanoTime,
+ pollResult);
+ lastSnapshot[0] = Snapshot.capture(stats);
+ lastReportNanoTime = nowNanoTime;
+ }
+ }
+
+ printReport(
+ "final",
+ Snapshot.zero(),
+ Snapshot.capture(stats),
+ System.nanoTime() - startNanoTime,
+ lastPollResult);
+ } finally {
+ if (Objects.nonNull(consumer)) {
+ consumer.close();
+ }
+ }
+ }
+
+ private static void createTopicIfNeeded(final PerfConfig config) throws Exception {
+ try (final ISubscriptionTreeSession session =
+ new SubscriptionTreeSessionBuilder()
+ .host(config.host)
+ .port(config.port)
+ .username(config.username)
+ .password(config.password)
+ .build()) {
+ session.open();
+
+ final Properties topicConfig = new Properties();
+ topicConfig.put(TopicConstant.MODE_KEY, TopicConstant.MODE_LIVE_VALUE);
+ topicConfig.put(TopicConstant.FORMAT_KEY, TopicConstant.FORMAT_RECORD_HANDLER_VALUE);
+ topicConfig.put(TopicConstant.PATH_KEY, config.path);
+ topicConfig.put(TopicConstant.ORDER_MODE_KEY, config.orderMode);
+ session.createTopicIfNotExists(config.topic, topicConfig);
+ }
+ }
+
+ private static SubscriptionTreePullConsumer createConsumer(final PerfConfig config) {
+ return (SubscriptionTreePullConsumer)
+ new SubscriptionTreePullConsumerBuilder()
+ .host(config.host)
+ .port(config.port)
+ .username(config.username)
+ .password(config.password)
+ .consumerId(config.consumer)
+ .consumerGroupId(config.group)
+ .autoCommit(config.autoCommit)
+ .autoCommitIntervalMs(config.autoCommitIntervalMs)
+ .maxPollParallelism(1)
+ .build();
+ }
+
+ private static SubscriptionTreePullConsumer openAndSubscribeConsumer(final PerfConfig config)
+ throws Exception {
+ final SubscriptionTreePullConsumer consumer = createConsumer(config);
+ consumer.open();
+ consumer.subscribe(config.topic);
+ return consumer;
+ }
+
+ private static PollResult emptyPollResult(final PerfStats stats) {
+ return new PollResult(Collections.emptyList(), 0, stats.lastWatermark);
+ }
+
+ private static boolean shouldStopConsumer(
+ final PerfConfig config,
+ final ConsumerRestartController controller,
+ final long elapsedNanoTime) {
+ return controller.enabled
+ && !controller.stopPerformed
+ && elapsedNanoTime >= config.consumerStopNanos;
+ }
+
+ private static boolean shouldResumeConsumer(
+ final PerfConfig config,
+ final ConsumerRestartController controller,
+ final long elapsedNanoTime) {
+ return controller.enabled
+ && controller.stopPerformed
+ && !controller.resumePerformed
+ && elapsedNanoTime >= config.consumerResumeNanos;
+ }
+
+ private static boolean shouldPauseConsumerByRows(
+ final PerfConfig config, final ConsumerPauseController controller, final long totalRows) {
+ return controller.enabled
+ && !controller.paused
+ && totalRows > 0
+ && totalRows >= controller.nextPauseRows
+ && config.consumerPauseEveryRows > 0;
+ }
+
+ private static boolean shouldResumeConsumerByRows(
+ final PerfConfig config, final ConsumerPauseController controller) {
+ return controller.enabled
+ && controller.paused
+ && controller.stoppedNanoTime > 0
+ && System.nanoTime() - controller.stoppedNanoTime >= config.consumerPauseDurationNanos;
+ }
+
+ private static void handlePollResult(
+ final PollResult pollResult,
+ final PerfStats stats,
+ final long processDelayNanos,
+ final ProcessingRateLimiter processingRateLimiter,
+ final String ingestWallTimeSensor) {
+ stats.totalPollCalls++;
+ stats.lastBufferedCount = pollResult.getBufferedCount();
+ if (pollResult.getWatermark() >= 0) {
+ stats.lastWatermark = pollResult.getWatermark();
+ }
+
+ final List messages = pollResult.getMessages();
+ if (messages.isEmpty()) {
+ stats.emptyPollCalls++;
+ return;
+ }
+
+ for (final SubscriptionMessage message : messages) {
+ stats.totalMessages++;
+
+ if (message.getMessageType() == SubscriptionMessageType.WATERMARK.getType()) {
+ stats.totalWatermarkMessages++;
+ if (message.getWatermarkTimestamp() >= 0) {
+ stats.lastWatermark = Math.max(stats.lastWatermark, message.getWatermarkTimestamp());
+ }
+ continue;
+ }
+
+ if (message.getMessageType() == SubscriptionMessageType.TS_FILE.getType()) {
+ stats.totalTsFileMessages++;
+ maybeApplyProcessingDelay(processDelayNanos, processingRateLimiter, 0);
+ continue;
+ }
+
+ if (message.getMessageType() == SubscriptionMessageType.RECORD_HANDLER.getType()) {
+ final Iterator tabletIterator = message.getRecordTabletIterator();
+ while (tabletIterator.hasNext()) {
+ final Tablet tablet = tabletIterator.next();
+ stats.totalTablets++;
+ final int rowSize = tablet.getRowSize();
+ stats.totalRows += rowSize;
+ stats.totalApproxBytes += tablet.ramBytesUsed();
+ updateOrderingStats(stats, tablet, rowSize);
+ updateLatencyStats(stats, tablet, rowSize, ingestWallTimeSensor);
+ maybeApplyProcessingDelay(
+ processDelayNanos, processingRateLimiter, estimateTabletPoints(tablet, rowSize));
+ }
+ }
+ }
+ }
+
+ private static long estimateTabletPoints(final Tablet tablet, final int rowSize) {
+ if (rowSize <= 0) {
+ return 0L;
+ }
+ return (long) rowSize * tablet.getSchemas().size();
+ }
+
+ private static void updateOrderingStats(
+ final PerfStats stats, final Tablet tablet, final int rowSize) {
+ if (rowSize <= 0) {
+ return;
+ }
+
+ final String deviceId = Objects.toString(tablet.getDeviceId(), "");
+ long lastSeenTimestamp = stats.lastSeenTimestampByDevice.getOrDefault(deviceId, Long.MIN_VALUE);
+
+ for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) {
+ final long currentTimestamp = tablet.getTimestamp(rowIndex);
+ if (stats.equivalentRowTracker == null
+ || stats.equivalentRowTracker.record(deviceId, currentTimestamp)) {
+ stats.totalEquivalentRows++;
+ }
+ if (lastSeenTimestamp != Long.MIN_VALUE && currentTimestamp < lastSeenTimestamp) {
+ stats.totalOutOfOrderRows++;
+ final long regression = lastSeenTimestamp - currentTimestamp;
+ if (regression > stats.maxTimestampRegression) {
+ stats.maxTimestampRegression = regression;
+ }
+ }
+ if (currentTimestamp > lastSeenTimestamp) {
+ lastSeenTimestamp = currentTimestamp;
+ }
+ }
+
+ stats.lastSeenTimestampByDevice.put(deviceId, lastSeenTimestamp);
+ }
+
+ private static void updateLatencyStats(
+ final PerfStats stats,
+ final Tablet tablet,
+ final int rowSize,
+ final String ingestWallTimeSensor) {
+ if (rowSize <= 0 || ingestWallTimeSensor == null || ingestWallTimeSensor.isEmpty()) {
+ return;
+ }
+
+ final int sensorIndex = findMeasurementIndex(tablet, ingestWallTimeSensor);
+ if (sensorIndex < 0) {
+ return;
+ }
+
+ final List schemas = tablet.getSchemas();
+ if (sensorIndex >= schemas.size()
+ || schemas.get(sensorIndex).getType() != TSDataType.INT64
+ || sensorIndex >= tablet.getValues().length
+ || !(tablet.getValues()[sensorIndex] instanceof long[])) {
+ return;
+ }
+
+ final long[] ingestWallTimes = (long[]) tablet.getValues()[sensorIndex];
+ final BitMap[] bitMaps = tablet.getBitMaps();
+ final BitMap bitMap =
+ bitMaps != null && sensorIndex < bitMaps.length ? bitMaps[sensorIndex] : null;
+ final long nowMs = System.currentTimeMillis();
+
+ for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) {
+ if (bitMap != null && bitMap.isMarked(rowIndex)) {
+ continue;
+ }
+
+ final long ingestWallTimeMs = ingestWallTimes[rowIndex];
+ final long latencyMs = Math.max(0L, nowMs - ingestWallTimeMs);
+ stats.recordLatency(latencyMs);
+ }
+ }
+
+ private static int findMeasurementIndex(final Tablet tablet, final String measurementName) {
+ final List schemas = tablet.getSchemas();
+ for (int i = 0, size = schemas.size(); i < size; i++) {
+ if (measurementName.equals(schemas.get(i).getMeasurementName())) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ private static void maybeApplyProcessingDelay(
+ final long processDelayNanos,
+ final ProcessingRateLimiter processingRateLimiter,
+ final long processedPoints) {
+ if (processingRateLimiter.isEnabled()) {
+ processingRateLimiter.acquire(processedPoints);
+ return;
+ }
+ if (processDelayNanos > 0) {
+ LockSupport.parkNanos(processDelayNanos);
+ }
+ }
+
+ private static void captureRandomSeekCheckpoint(
+ final SubscriptionTreePullConsumer consumer,
+ final PerfConfig config,
+ final PerfStats stats,
+ final RandomSeekController controller)
+ throws Exception {
+ if (!controller.enabled
+ || stats.totalRows <= 0
+ || (controller.lastCapturedRows >= 0
+ && stats.totalRows - controller.lastCapturedRows
+ < RANDOM_SEEK_CHECKPOINT_INTERVAL_ROWS)) {
+ return;
+ }
+
+ TopicProgress progress = consumer.committedPositions(config.topic);
+ String source = "committed";
+ if (isEmptyTopicProgress(progress)) {
+ progress = consumer.positions(config.topic);
+ source = "current";
+ }
+ if (isEmptyTopicProgress(progress)) {
+ return;
+ }
+
+ final TopicProgress safeProgress = new TopicProgress(progress.getRegionProgress());
+ if (Objects.equals(controller.lastCapturedProgress, safeProgress)) {
+ controller.lastCapturedRows = stats.totalRows;
+ return;
+ }
+
+ controller.checkpoints.add(
+ new SeekCheckpoint(stats.totalRows, stats.totalEquivalentRows, source, safeProgress));
+ controller.lastCapturedRows = stats.totalRows;
+ controller.lastCapturedProgress = safeProgress;
+ stats.totalRandomSeekCheckpoints = controller.checkpoints.size();
+ }
+
+ private static void captureScheduledSeekCheckpoint(
+ final SubscriptionTreePullConsumer consumer,
+ final PerfConfig config,
+ final PerfStats stats,
+ final ScheduledSeekController controller)
+ throws Exception {
+ if (!controller.enabled || Objects.nonNull(controller.checkpoint)) {
+ return;
+ }
+ if (stats.totalRows < config.seekCaptureRows) {
+ return;
+ }
+
+ final TopicProgress currentProgress = consumer.positions(config.topic);
+ if (isEmptyTopicProgress(currentProgress)) {
+ return;
+ }
+ final TopicProgress committedProgress = consumer.committedPositions(config.topic);
+
+ controller.checkpoint =
+ new SeekCheckpoint(
+ stats.totalRows,
+ stats.totalEquivalentRows,
+ "current",
+ new TopicProgress(currentProgress.getRegionProgress()));
+
+ System.out.println(
+ String.format(
+ Locale.ROOT,
+ "[%s] Scheduled seek checkpoint captured: checkpointRows=%d, checkpointEquivalentRows=%d, progressSource=current, triggerSec=%.3f",
+ nowString(),
+ controller.checkpoint.rawRows,
+ controller.checkpoint.equivalentRows,
+ config.seekTriggerSec));
+ }
+
+ private static void maybePerformRandomSeek(
+ final SubscriptionTreePullConsumer consumer,
+ final PerfConfig config,
+ final PerfStats stats,
+ final RandomSeekController controller)
+ throws Exception {
+ if (!controller.enabled
+ || controller.performed
+ || stats.totalRows < config.randomSeekMinRows
+ || controller.checkpoints.size() < 2) {
+ return;
+ }
+
+ final int candidateCount = controller.checkpoints.size() - 1;
+ final SeekCheckpoint targetCheckpoint =
+ controller.checkpoints.get(controller.random.nextInt(candidateCount));
+
+ consumer.seekAfter(config.topic, targetCheckpoint.topicProgress);
+
+ controller.performed = true;
+ stats.totalRandomSeeks++;
+ stats.lastRandomSeekSourceRows = targetCheckpoint.rawRows;
+ stats.lastRandomSeekEquivalentRows = targetCheckpoint.equivalentRows;
+ stats.lastRandomSeekObservedRows = stats.totalRows;
+
+ System.out.println(
+ String.format(
+ Locale.ROOT,
+ "[%s] Random seekAfter triggered: checkpointRows=%d, checkpointEquivalentRows=%d, progressSource=%s, checkpointCount=%d",
+ nowString(),
+ targetCheckpoint.rawRows,
+ targetCheckpoint.equivalentRows,
+ targetCheckpoint.source,
+ controller.checkpoints.size()));
+ }
+
+ private static void maybePerformScheduledSeek(
+ final SubscriptionTreePullConsumer consumer,
+ final PerfConfig config,
+ final PerfStats stats,
+ final ScheduledSeekController controller,
+ final long elapsedNanoTime)
+ throws Exception {
+ if (!controller.enabled
+ || controller.performed
+ || Objects.isNull(controller.checkpoint)
+ || elapsedNanoTime < config.seekTriggerNanos) {
+ return;
+ }
+
+ consumer.seekAfter(config.topic, controller.checkpoint.topicProgress);
+
+ controller.performed = true;
+ stats.totalRandomSeeks++;
+
+ System.out.println(
+ String.format(
+ Locale.ROOT,
+ "[%s] Scheduled seekAfter triggered: checkpointRows=%d, checkpointEquivalentRows=%d, progressSource=%s, triggerSec=%.3f",
+ nowString(),
+ controller.checkpoint.rawRows,
+ controller.checkpoint.equivalentRows,
+ controller.checkpoint.source,
+ config.seekTriggerSec));
+ }
+
+ private static boolean isEmptyTopicProgress(final TopicProgress topicProgress) {
+ return Objects.isNull(topicProgress) || topicProgress.getRegionProgress().isEmpty();
+ }
+
+ private static void printReport(
+ final String label,
+ final Snapshot previous,
+ final Snapshot current,
+ final long elapsedNanoTime,
+ final PollResult pollResult) {
+ final double seconds = Math.max(1e-9, elapsedNanoTime / 1_000_000_000.0d);
+
+ final long intervalMessages = current.totalMessages - previous.totalMessages;
+ final long intervalTablets = current.totalTablets - previous.totalTablets;
+ final long intervalRows = current.totalRows - previous.totalRows;
+ final long intervalEquivalentRows = current.totalEquivalentRows - previous.totalEquivalentRows;
+ final long intervalBytes = current.totalApproxBytes - previous.totalApproxBytes;
+ final long intervalWatermarks =
+ current.totalWatermarkMessages - previous.totalWatermarkMessages;
+ final long intervalOutOfOrderRows = current.totalOutOfOrderRows - previous.totalOutOfOrderRows;
+ final double intervalOutOfOrderRatio =
+ intervalRows <= 0 ? 0d : intervalOutOfOrderRows * 100.0d / intervalRows;
+ final double totalOutOfOrderRatio =
+ current.totalRows <= 0 ? 0d : current.totalOutOfOrderRows * 100.0d / current.totalRows;
+ final LatencySummary intervalLatency = LatencySummary.delta(previous, current);
+ final LatencySummary totalLatency = LatencySummary.total(current);
+
+ System.out.println(
+ String.format(
+ Locale.ROOT,
+ "[%s] %-8s msgs=%d (%.1f/s), tablets=%d (%.1f/s), rows=%d (%.1f/s), eqRows=%d (%.1f/s), bytes=%s (%s/s), "
+ + "watermarks=%d, oooRows=%d (%.4f%%), totalOoo=%.4f%%, maxTsBack=%d, "
+ + "latRows=%d, latAvgMs=%s, latP95Ms=%s, latP99Ms=%s, latMaxMs=%s, totalLatAvgMs=%s, totalLatP95Ms=%s, totalLatP99Ms=%s, totalLatMaxMs=%s, "
+ + "totalRows=%d, equivalentRows=%d, replayRows=%d, seeks=%d, totalBytes=%s, polls=%d, emptyPolls=%d, buffered=%d, watermark=%s",
+ nowString(),
+ label,
+ intervalMessages,
+ intervalMessages / seconds,
+ intervalTablets,
+ intervalTablets / seconds,
+ intervalRows,
+ intervalRows / seconds,
+ intervalEquivalentRows,
+ intervalEquivalentRows / seconds,
+ formatBytes(intervalBytes),
+ formatBytes((long) (intervalBytes / seconds)),
+ intervalWatermarks,
+ intervalOutOfOrderRows,
+ intervalOutOfOrderRatio,
+ totalOutOfOrderRatio,
+ current.maxTimestampRegression,
+ intervalLatency.sampleCount,
+ intervalLatency.formatAverageMs(),
+ intervalLatency.p95MsLabel,
+ intervalLatency.p99MsLabel,
+ intervalLatency.maxMsLabel,
+ totalLatency.formatAverageMs(),
+ totalLatency.p95MsLabel,
+ totalLatency.p99MsLabel,
+ totalLatency.maxMsLabel,
+ current.totalRows,
+ current.totalEquivalentRows,
+ current.totalRows - current.totalEquivalentRows,
+ current.totalRandomSeeks,
+ formatBytes(current.totalApproxBytes),
+ current.totalPollCalls,
+ current.emptyPollCalls,
+ pollResult.getBufferedCount(),
+ formatWatermark(current.lastWatermark)));
+ }
+
+ private static String formatWatermark(final long watermark) {
+ return watermark >= 0 ? Long.toString(watermark) : "N/A";
+ }
+
+ private static String formatBytes(final long bytes) {
+ final long absBytes = Math.abs(bytes);
+ if (absBytes < 1024) {
+ return bytes + " B";
+ }
+ if (absBytes < 1024L * 1024) {
+ return String.format(Locale.ROOT, "%.2f KiB", bytes / 1024.0d);
+ }
+ if (absBytes < 1024L * 1024 * 1024) {
+ return String.format(Locale.ROOT, "%.2f MiB", bytes / 1024.0d / 1024.0d);
+ }
+ return String.format(Locale.ROOT, "%.2f GiB", bytes / 1024.0d / 1024.0d / 1024.0d);
+ }
+
+ private static String nowString() {
+ return TIME_FORMATTER.format(Instant.now());
+ }
+
+ private static long nanosToSeconds(final long nanos) {
+ return nanos / 1_000_000_000L;
+ }
+
+ private static void printUsage() {
+ System.out.println("Usage:");
+ System.out.println(
+ " java ... org.apache.iotdb.ConsensusSubscriptionPerfTest [--key=value ...]");
+ System.out.println();
+ System.out.println("Available keys:");
+ System.out.println(" host=127.0.0.1");
+ System.out.println(" port=6667");
+ System.out.println(" username=root");
+ System.out.println(" password=root");
+ System.out.println(" topic=topic_perf_");
+ System.out.println(" group=cg_perf_");
+ System.out.println(" consumer=consumer_perf_");
+ System.out.println(" path=root.**");
+ System.out.println(" orderMode=leader-only");
+ System.out.println(" autoCreateTopic=true");
+ System.out.println(" createTopicOnly=false");
+ System.out.println(" autoCommit=true");
+ System.out.println(" autoCommitIntervalMs=1000");
+ System.out.println(" pollTimeoutMs=1000");
+ System.out.println(" waitBeforePollSec=0");
+ System.out.println(" reportIntervalSec=5");
+ System.out.println(" durationSec=0 (0 means run until manually stopped)");
+ System.out.println(" processDelayMs=0 (delay per non-watermark message, decimal allowed)");
+ System.out.println(" targetPointsPerSec=0 (0 disables point-rate limiting)");
+ System.out.println(" ingestWallTimeSensor=ingest_wall_time_ms");
+ System.out.println(" randomSeek=false");
+ System.out.println(" randomSeekMinRows=1000000");
+ System.out.println(" seekCaptureRows=0 (0 disables scheduled checkpoint capture)");
+ System.out.println(" seekTriggerSec=0 (0 disables scheduled seek)");
+ System.out.println(
+ " consumerStopSec=0 (0 disables consumer polling pause/resume simulation)");
+ System.out.println(" consumerResumeSec=0 (must be > consumerStopSec when enabled)");
+ System.out.println(" consumerPauseEveryRows=0 (0 disables row-based recurring pauses)");
+ System.out.println(
+ " consumerPauseDurationSec=0 (must be > 0 when consumerPauseEveryRows is enabled)");
+ }
+
+ private static final class PerfConfig {
+ private final boolean help;
+ private final String host;
+ private final int port;
+ private final String username;
+ private final String password;
+ private final String topic;
+ private final String group;
+ private final String consumer;
+ private final String path;
+ private final String orderMode;
+ private final String ingestWallTimeSensor;
+ private final boolean autoCreateTopic;
+ private final boolean createTopicOnly;
+ private final boolean autoCommit;
+ private final long autoCommitIntervalMs;
+ private final long pollTimeoutMs;
+ private final double waitBeforePollSec;
+ private final long waitBeforePollNanos;
+ private final long reportIntervalSec;
+ private final long durationSec;
+ private final double processDelayMs;
+ private final long processDelayNanos;
+ private final double targetPointsPerSec;
+ private final boolean randomSeek;
+ private final long randomSeekMinRows;
+ private final long seekCaptureRows;
+ private final double seekTriggerSec;
+ private final long seekTriggerNanos;
+ private final double consumerStopSec;
+ private final long consumerStopNanos;
+ private final double consumerResumeSec;
+ private final long consumerResumeNanos;
+ private final long consumerPauseEveryRows;
+ private final double consumerPauseDurationSec;
+ private final long consumerPauseDurationNanos;
+
+ private PerfConfig(
+ final boolean help,
+ final String host,
+ final int port,
+ final String username,
+ final String password,
+ final String topic,
+ final String group,
+ final String consumer,
+ final String path,
+ final String orderMode,
+ final String ingestWallTimeSensor,
+ final boolean autoCreateTopic,
+ final boolean createTopicOnly,
+ final boolean autoCommit,
+ final long autoCommitIntervalMs,
+ final long pollTimeoutMs,
+ final double waitBeforePollSec,
+ final long waitBeforePollNanos,
+ final long reportIntervalSec,
+ final long durationSec,
+ final double processDelayMs,
+ final long processDelayNanos,
+ final double targetPointsPerSec,
+ final boolean randomSeek,
+ final long randomSeekMinRows,
+ final long seekCaptureRows,
+ final double seekTriggerSec,
+ final long seekTriggerNanos,
+ final double consumerStopSec,
+ final long consumerStopNanos,
+ final double consumerResumeSec,
+ final long consumerResumeNanos,
+ final long consumerPauseEveryRows,
+ final double consumerPauseDurationSec,
+ final long consumerPauseDurationNanos) {
+ this.help = help;
+ this.host = host;
+ this.port = port;
+ this.username = username;
+ this.password = password;
+ this.topic = topic;
+ this.group = group;
+ this.consumer = consumer;
+ this.path = path;
+ this.orderMode = orderMode;
+ this.ingestWallTimeSensor = ingestWallTimeSensor;
+ this.autoCreateTopic = autoCreateTopic;
+ this.createTopicOnly = createTopicOnly;
+ this.autoCommit = autoCommit;
+ this.autoCommitIntervalMs = autoCommitIntervalMs;
+ this.pollTimeoutMs = pollTimeoutMs;
+ this.waitBeforePollSec = waitBeforePollSec;
+ this.waitBeforePollNanos = waitBeforePollNanos;
+ this.reportIntervalSec = reportIntervalSec;
+ this.durationSec = durationSec;
+ this.processDelayMs = processDelayMs;
+ this.processDelayNanos = processDelayNanos;
+ this.targetPointsPerSec = targetPointsPerSec;
+ this.randomSeek = randomSeek;
+ this.randomSeekMinRows = randomSeekMinRows;
+ this.seekCaptureRows = seekCaptureRows;
+ this.seekTriggerSec = seekTriggerSec;
+ this.seekTriggerNanos = seekTriggerNanos;
+ this.consumerStopSec = consumerStopSec;
+ this.consumerStopNanos = consumerStopNanos;
+ this.consumerResumeSec = consumerResumeSec;
+ this.consumerResumeNanos = consumerResumeNanos;
+ this.consumerPauseEveryRows = consumerPauseEveryRows;
+ this.consumerPauseDurationSec = consumerPauseDurationSec;
+ this.consumerPauseDurationNanos = consumerPauseDurationNanos;
+ }
+
+ private static PerfConfig parse(final String[] args) {
+ final long suffix = System.currentTimeMillis();
+ String host = "127.0.0.1";
+ int port = 6667;
+ String username = "root";
+ String password = "root";
+ String topic = "topic_perf_" + suffix;
+ String group = "cg_perf_" + suffix;
+ String consumer = "consumer_perf_" + suffix;
+ String path = "root.**";
+ String orderMode = TopicConstant.ORDER_MODE_DEFAULT_VALUE;
+ orderMode = TopicConstant.ORDER_MODE_PER_WRITER_VALUE;
+ String ingestWallTimeSensor = "ingest_wall_time_ms";
+ boolean autoCreateTopic = true;
+ boolean createTopicOnly = false;
+ boolean autoCommit = true;
+ long autoCommitIntervalMs = 1000L;
+ long pollTimeoutMs = 1000L;
+ double waitBeforePollSec = 0d;
+ long reportIntervalSec = 1L;
+ long durationSec = 0L;
+ double processDelayMs = 0d;
+ double targetPointsPerSec = 10_000_000d;
+ boolean randomSeek = false;
+ long randomSeekMinRows = 2_000_000L;
+ long seekCaptureRows = 0L;
+ double seekTriggerSec = 0d;
+ double consumerStopSec = 0d;
+ double consumerResumeSec = 0d;
+ long consumerPauseEveryRows = 0L;
+ double consumerPauseDurationSec = 0d;
+ boolean help = false;
+
+ for (final String arg : args) {
+ if ("--help".equals(arg) || "-h".equals(arg)) {
+ help = true;
+ continue;
+ }
+
+ final String normalized = arg.startsWith("--") ? arg.substring(2) : arg;
+ final int separator = normalized.indexOf('=');
+ if (separator <= 0) {
+ throw new IllegalArgumentException(
+ "Invalid argument: " + arg + ". Expected format --key=value");
+ }
+
+ final String key = normalized.substring(0, separator);
+ final String value = normalized.substring(separator + 1);
+
+ switch (key) {
+ case "host":
+ host = value;
+ break;
+ case "port":
+ port = Integer.parseInt(value);
+ break;
+ case "username":
+ username = value;
+ break;
+ case "password":
+ password = value;
+ break;
+ case "topic":
+ topic = value;
+ break;
+ case "group":
+ group = value;
+ break;
+ case "consumer":
+ consumer = value;
+ break;
+ case "path":
+ path = value;
+ break;
+ case "orderMode":
+ case "order-mode":
+ orderMode = TopicConfig.normalizeOrderMode(value);
+ break;
+ case "ingestWallTimeSensor":
+ case "ingest-wall-time-sensor":
+ ingestWallTimeSensor = value;
+ break;
+ case "autoCreateTopic":
+ autoCreateTopic = Boolean.parseBoolean(value);
+ break;
+ case "createTopicOnly":
+ createTopicOnly = Boolean.parseBoolean(value);
+ break;
+ case "autoCommit":
+ autoCommit = Boolean.parseBoolean(value);
+ break;
+ case "autoCommitIntervalMs":
+ autoCommitIntervalMs = Long.parseLong(value);
+ break;
+ case "pollTimeoutMs":
+ pollTimeoutMs = Long.parseLong(value);
+ break;
+ case "waitBeforePollSec":
+ waitBeforePollSec = Double.parseDouble(value);
+ break;
+ case "reportIntervalSec":
+ reportIntervalSec = Long.parseLong(value);
+ break;
+ case "durationSec":
+ durationSec = Long.parseLong(value);
+ break;
+ case "processDelayMs":
+ processDelayMs = Double.parseDouble(value);
+ break;
+ case "targetPointsPerSec":
+ case "target-points-per-sec":
+ targetPointsPerSec = Double.parseDouble(value);
+ break;
+ case "randomSeek":
+ randomSeek = Boolean.parseBoolean(value);
+ break;
+ case "randomSeekMinRows":
+ randomSeekMinRows = Long.parseLong(value);
+ break;
+ case "seekCaptureRows":
+ seekCaptureRows = Long.parseLong(value);
+ break;
+ case "seekTriggerSec":
+ seekTriggerSec = Double.parseDouble(value);
+ break;
+ case "consumerStopSec":
+ case "consumer-stop-sec":
+ consumerStopSec = Double.parseDouble(value);
+ break;
+ case "consumerResumeSec":
+ case "consumer-resume-sec":
+ consumerResumeSec = Double.parseDouble(value);
+ break;
+ case "consumerPauseEveryRows":
+ case "consumer-pause-every-rows":
+ consumerPauseEveryRows = Long.parseLong(value);
+ break;
+ case "consumerPauseDurationSec":
+ case "consumer-pause-duration-sec":
+ consumerPauseDurationSec = Double.parseDouble(value);
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown argument key: " + key);
+ }
+ }
+
+ if (!TopicConfig.isValidOrderMode(orderMode)) {
+ throw new IllegalArgumentException("Unsupported orderMode: " + orderMode);
+ }
+ if (processDelayMs < 0) {
+ throw new IllegalArgumentException("processDelayMs must be >= 0");
+ }
+ if (targetPointsPerSec < 0) {
+ throw new IllegalArgumentException("targetPointsPerSec must be >= 0");
+ }
+ if (waitBeforePollSec < 0) {
+ throw new IllegalArgumentException("waitBeforePollSec must be >= 0");
+ }
+ if (randomSeekMinRows < 0) {
+ throw new IllegalArgumentException("randomSeekMinRows must be >= 0");
+ }
+ if (seekCaptureRows < 0) {
+ throw new IllegalArgumentException("seekCaptureRows must be >= 0");
+ }
+ if (seekTriggerSec < 0) {
+ throw new IllegalArgumentException("seekTriggerSec must be >= 0");
+ }
+ if (consumerStopSec < 0) {
+ throw new IllegalArgumentException("consumerStopSec must be >= 0");
+ }
+ if (consumerResumeSec < 0) {
+ throw new IllegalArgumentException("consumerResumeSec must be >= 0");
+ }
+ if (consumerPauseEveryRows < 0) {
+ throw new IllegalArgumentException("consumerPauseEveryRows must be >= 0");
+ }
+ if (consumerPauseDurationSec < 0) {
+ throw new IllegalArgumentException("consumerPauseDurationSec must be >= 0");
+ }
+ if ((seekCaptureRows > 0) != (seekTriggerSec > 0)) {
+ throw new IllegalArgumentException(
+ "seekCaptureRows and seekTriggerSec must both be set to positive values to enable scheduled seek");
+ }
+ if ((consumerStopSec > 0) != (consumerResumeSec > 0)) {
+ throw new IllegalArgumentException(
+ "consumerStopSec and consumerResumeSec must both be set to positive values to enable consumer polling pause/resume simulation");
+ }
+ if (consumerResumeSec > 0 && consumerResumeSec <= consumerStopSec) {
+ throw new IllegalArgumentException(
+ "consumerResumeSec must be greater than consumerStopSec");
+ }
+ if ((consumerPauseEveryRows > 0) != (consumerPauseDurationSec > 0)) {
+ throw new IllegalArgumentException(
+ "consumerPauseEveryRows and consumerPauseDurationSec must both be set to positive values to enable row-based recurring pauses");
+ }
+ if (consumerPauseEveryRows > 0 && consumerStopSec > 0) {
+ throw new IllegalArgumentException(
+ "consumerPauseEveryRows/consumerPauseDurationSec cannot be combined with consumerStopSec/consumerResumeSec");
+ }
+
+ final long waitBeforePollNanos = Math.round(waitBeforePollSec * 1_000_000_000.0d);
+ final long processDelayNanos = Math.round(processDelayMs * 1_000_000.0d);
+ final long seekTriggerNanos = Math.round(seekTriggerSec * 1_000_000_000.0d);
+ final long consumerStopNanos = Math.round(consumerStopSec * 1_000_000_000.0d);
+ final long consumerResumeNanos = Math.round(consumerResumeSec * 1_000_000_000.0d);
+ final long consumerPauseDurationNanos =
+ Math.round(consumerPauseDurationSec * 1_000_000_000.0d);
+
+ return new PerfConfig(
+ help,
+ host,
+ port,
+ username,
+ password,
+ topic,
+ group,
+ consumer,
+ path,
+ orderMode,
+ ingestWallTimeSensor,
+ autoCreateTopic,
+ createTopicOnly,
+ autoCommit,
+ autoCommitIntervalMs,
+ pollTimeoutMs,
+ waitBeforePollSec,
+ waitBeforePollNanos,
+ reportIntervalSec,
+ durationSec,
+ processDelayMs,
+ processDelayNanos,
+ targetPointsPerSec,
+ randomSeek,
+ randomSeekMinRows,
+ seekCaptureRows,
+ seekTriggerSec,
+ seekTriggerNanos,
+ consumerStopSec,
+ consumerStopNanos,
+ consumerResumeSec,
+ consumerResumeNanos,
+ consumerPauseEveryRows,
+ consumerPauseDurationSec,
+ consumerPauseDurationNanos);
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ Locale.ROOT,
+ "Config{host=%s, port=%d, username=%s, topic=%s, group=%s, consumer=%s, path=%s, "
+ + "orderMode=%s, ingestWallTimeSensor=%s, autoCreateTopic=%s, createTopicOnly=%s, autoCommit=%s, autoCommitIntervalMs=%d, pollTimeoutMs=%d, "
+ + "waitBeforePollSec=%.3f, "
+ + "reportIntervalSec=%d, durationSec=%d, processDelayMs=%.3f, targetPointsPerSec=%.3f, randomSeek=%s, randomSeekMinRows=%d, seekCaptureRows=%d, seekTriggerSec=%.3f, consumerStopSec=%.3f, consumerResumeSec=%.3f, consumerPauseEveryRows=%d, consumerPauseDurationSec=%.3f}",
+ host,
+ port,
+ username,
+ topic,
+ group,
+ consumer,
+ path,
+ orderMode,
+ ingestWallTimeSensor,
+ autoCreateTopic,
+ createTopicOnly,
+ autoCommit,
+ autoCommitIntervalMs,
+ pollTimeoutMs,
+ waitBeforePollSec,
+ reportIntervalSec,
+ durationSec,
+ processDelayMs,
+ targetPointsPerSec,
+ randomSeek,
+ randomSeekMinRows,
+ seekCaptureRows,
+ seekTriggerSec,
+ consumerStopSec,
+ consumerResumeSec,
+ consumerPauseEveryRows,
+ consumerPauseDurationSec);
+ }
+
+ private boolean enableEquivalentRowTracking() {
+ return randomSeek || (seekCaptureRows > 0 && seekTriggerSec > 0);
+ }
+ }
+
+ private static final class ProcessingRateLimiter {
+ private final double targetPointsPerSec;
+ private long throttlingStartNanoTime = -1L;
+ private long totalProcessedPoints = 0L;
+
+ private ProcessingRateLimiter(final double targetPointsPerSec) {
+ this.targetPointsPerSec = targetPointsPerSec;
+ }
+
+ private boolean isEnabled() {
+ return targetPointsPerSec > 0d;
+ }
+
+ private void acquire(final long processedPoints) {
+ if (!isEnabled() || processedPoints <= 0) {
+ return;
+ }
+
+ final long nowNanoTime = System.nanoTime();
+ if (throttlingStartNanoTime < 0) {
+ throttlingStartNanoTime = nowNanoTime;
+ }
+
+ totalProcessedPoints += processedPoints;
+ final long targetElapsedNanos =
+ (long) Math.ceil((totalProcessedPoints * 1_000_000_000.0d) / targetPointsPerSec);
+ final long actualElapsedNanos = nowNanoTime - throttlingStartNanoTime;
+ final long remainingNanos = targetElapsedNanos - actualElapsedNanos;
+ if (remainingNanos > 0) {
+ LockSupport.parkNanos(remainingNanos);
+ }
+ }
+
+ private void pauseForDowntime(final long pausedNanos) {
+ if (!isEnabled() || throttlingStartNanoTime < 0 || pausedNanos <= 0) {
+ return;
+ }
+ throttlingStartNanoTime += pausedNanos;
+ }
+ }
+
+ private static final class PerfStats {
+ private long totalPollCalls;
+ private long emptyPollCalls;
+ private long totalMessages;
+ private long totalWatermarkMessages;
+ private long totalTsFileMessages;
+ private long totalTablets;
+ private long totalRows;
+ private long totalEquivalentRows;
+ private long totalApproxBytes;
+ private long totalOutOfOrderRows;
+ private long maxTimestampRegression;
+ private long totalLatencySamples;
+ private long totalLatencySumMs;
+ private final long[] latencyHistogramBuckets = new long[LatencyHistogram.BUCKET_COUNT];
+ private int lastBufferedCount;
+ private long lastWatermark = -1L;
+ private final Map lastSeenTimestampByDevice = new HashMap<>();
+ private final EquivalentRowTracker equivalentRowTracker;
+ private long totalRandomSeeks;
+ private long totalRandomSeekCheckpoints;
+ private long lastRandomSeekSourceRows = -1L;
+ private long lastRandomSeekEquivalentRows = -1L;
+ private long lastRandomSeekObservedRows = -1L;
+
+ private PerfStats(final boolean enableEquivalentRowTracking) {
+ this.equivalentRowTracker = enableEquivalentRowTracking ? new EquivalentRowTracker() : null;
+ }
+
+ private void recordLatency(final long latencyMs) {
+ totalLatencySamples++;
+ totalLatencySumMs += latencyMs;
+ latencyHistogramBuckets[LatencyHistogram.bucketIndex(latencyMs)]++;
+ }
+ }
+
+ private static final class Snapshot {
+ private final long totalPollCalls;
+ private final long emptyPollCalls;
+ private final long totalMessages;
+ private final long totalWatermarkMessages;
+ private final long totalTablets;
+ private final long totalRows;
+ private final long totalEquivalentRows;
+ private final long totalApproxBytes;
+ private final long totalOutOfOrderRows;
+ private final long maxTimestampRegression;
+ private final long totalLatencySamples;
+ private final long totalLatencySumMs;
+ private final long[] latencyHistogramBuckets;
+ private final long lastWatermark;
+ private final long totalRandomSeeks;
+
+ private Snapshot(
+ final long totalPollCalls,
+ final long emptyPollCalls,
+ final long totalMessages,
+ final long totalWatermarkMessages,
+ final long totalTablets,
+ final long totalRows,
+ final long totalEquivalentRows,
+ final long totalApproxBytes,
+ final long totalOutOfOrderRows,
+ final long maxTimestampRegression,
+ final long totalLatencySamples,
+ final long totalLatencySumMs,
+ final long[] latencyHistogramBuckets,
+ final long lastWatermark,
+ final long totalRandomSeeks) {
+ this.totalPollCalls = totalPollCalls;
+ this.emptyPollCalls = emptyPollCalls;
+ this.totalMessages = totalMessages;
+ this.totalWatermarkMessages = totalWatermarkMessages;
+ this.totalTablets = totalTablets;
+ this.totalRows = totalRows;
+ this.totalEquivalentRows = totalEquivalentRows;
+ this.totalApproxBytes = totalApproxBytes;
+ this.totalOutOfOrderRows = totalOutOfOrderRows;
+ this.maxTimestampRegression = maxTimestampRegression;
+ this.totalLatencySamples = totalLatencySamples;
+ this.totalLatencySumMs = totalLatencySumMs;
+ this.latencyHistogramBuckets = latencyHistogramBuckets;
+ this.lastWatermark = lastWatermark;
+ this.totalRandomSeeks = totalRandomSeeks;
+ }
+
+ private static Snapshot capture(final PerfStats stats) {
+ Objects.requireNonNull(stats, "stats");
+ return new Snapshot(
+ stats.totalPollCalls,
+ stats.emptyPollCalls,
+ stats.totalMessages,
+ stats.totalWatermarkMessages,
+ stats.totalTablets,
+ stats.totalRows,
+ stats.totalEquivalentRows,
+ stats.totalApproxBytes,
+ stats.totalOutOfOrderRows,
+ stats.maxTimestampRegression,
+ stats.totalLatencySamples,
+ stats.totalLatencySumMs,
+ Arrays.copyOf(stats.latencyHistogramBuckets, stats.latencyHistogramBuckets.length),
+ stats.lastWatermark,
+ stats.totalRandomSeeks);
+ }
+
+ private static Snapshot zero() {
+ return new Snapshot(
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, new long[LatencyHistogram.BUCKET_COUNT], -1L, 0);
+ }
+ }
+
+ private static final class RandomSeekController {
+ private final boolean enabled;
+ private final Random random = new Random();
+ private final List checkpoints = new ArrayList<>();
+ private boolean performed;
+ private long lastCapturedRows = Long.MIN_VALUE;
+ private TopicProgress lastCapturedProgress;
+
+ private RandomSeekController(final boolean enabled) {
+ this.enabled = enabled;
+ }
+ }
+
+ private static final class ScheduledSeekController {
+ private final boolean enabled;
+ private boolean performed;
+ private SeekCheckpoint checkpoint;
+
+ private ScheduledSeekController(final boolean enabled) {
+ this.enabled = enabled;
+ }
+ }
+
+ private static final class ConsumerRestartController {
+ private final boolean enabled;
+ private boolean stopPerformed;
+ private boolean resumePerformed;
+ private long stoppedNanoTime = -1L;
+
+ private ConsumerRestartController(final boolean enabled) {
+ this.enabled = enabled;
+ }
+ }
+
+ private static final class ConsumerPauseController {
+ private final boolean enabled;
+ private long nextPauseRows;
+ private boolean paused;
+ private long stoppedNanoTime = -1L;
+ private long pausePerformedCount;
+
+ private ConsumerPauseController(final long pauseEveryRows) {
+ this.enabled = pauseEveryRows > 0;
+ this.nextPauseRows = pauseEveryRows;
+ }
+ }
+
+ private static final class SeekCheckpoint {
+ private final long rawRows;
+ private final long equivalentRows;
+ private final String source;
+ private final TopicProgress topicProgress;
+
+ private SeekCheckpoint(
+ final long rawRows,
+ final long equivalentRows,
+ final String source,
+ final TopicProgress topicProgress) {
+ this.rawRows = rawRows;
+ this.equivalentRows = equivalentRows;
+ this.source = source;
+ this.topicProgress = topicProgress;
+ }
+ }
+
+ private static final class EquivalentRowTracker {
+ private final Map> intervalsByDevice = new HashMap<>();
+
+ private boolean record(final String deviceId, final long timestamp) {
+ final NavigableMap intervals =
+ intervalsByDevice.computeIfAbsent(deviceId, ignored -> new TreeMap<>());
+ final Map.Entry floor = intervals.floorEntry(timestamp);
+ if (Objects.nonNull(floor) && floor.getValue() >= timestamp) {
+ return false;
+ }
+
+ long start = timestamp;
+ long end = timestamp;
+
+ if (Objects.nonNull(floor) && floor.getValue() + 1 == timestamp) {
+ start = floor.getKey();
+ intervals.remove(floor.getKey());
+ }
+
+ final Map.Entry ceiling = intervals.ceilingEntry(timestamp);
+ if (Objects.nonNull(ceiling) && ceiling.getKey() - 1 == timestamp) {
+ end = ceiling.getValue();
+ intervals.remove(ceiling.getKey());
+ }
+
+ intervals.put(start, end);
+ return true;
+ }
+ }
+
+ private static final class LatencyHistogram {
+ private static final int MAX_TRACKED_LATENCY_MS = 60_000;
+ private static final int BUCKET_COUNT = MAX_TRACKED_LATENCY_MS + 2;
+
+ private static int bucketIndex(final long latencyMs) {
+ if (latencyMs <= 0) {
+ return 0;
+ }
+ if (latencyMs > MAX_TRACKED_LATENCY_MS) {
+ return MAX_TRACKED_LATENCY_MS + 1;
+ }
+ return (int) latencyMs;
+ }
+
+ private static String bucketLabel(final int bucketIndex) {
+ if (bucketIndex > MAX_TRACKED_LATENCY_MS) {
+ return ">" + MAX_TRACKED_LATENCY_MS;
+ }
+ return Integer.toString(bucketIndex);
+ }
+ }
+
+ private static final class LatencySummary {
+ private final long sampleCount;
+ private final long sumMs;
+ private final String p95MsLabel;
+ private final String p99MsLabel;
+ private final String maxMsLabel;
+
+ private LatencySummary(
+ final long sampleCount,
+ final long sumMs,
+ final String p95MsLabel,
+ final String p99MsLabel,
+ final String maxMsLabel) {
+ this.sampleCount = sampleCount;
+ this.sumMs = sumMs;
+ this.p95MsLabel = p95MsLabel;
+ this.p99MsLabel = p99MsLabel;
+ this.maxMsLabel = maxMsLabel;
+ }
+
+ private static LatencySummary delta(final Snapshot previous, final Snapshot current) {
+ final long sampleCount = current.totalLatencySamples - previous.totalLatencySamples;
+ final long sumMs = current.totalLatencySumMs - previous.totalLatencySumMs;
+ if (sampleCount <= 0) {
+ return empty();
+ }
+ return summarize(
+ sampleCount, sumMs, current.latencyHistogramBuckets, previous.latencyHistogramBuckets);
+ }
+
+ private static LatencySummary total(final Snapshot current) {
+ if (current.totalLatencySamples <= 0) {
+ return empty();
+ }
+ return summarize(
+ current.totalLatencySamples,
+ current.totalLatencySumMs,
+ current.latencyHistogramBuckets,
+ null);
+ }
+
+ private static LatencySummary summarize(
+ final long sampleCount,
+ final long sumMs,
+ final long[] currentBuckets,
+ final long[] previousBuckets) {
+ final long p95Threshold = Math.max(1L, (long) Math.ceil(sampleCount * 0.95d));
+ final long p99Threshold = Math.max(1L, (long) Math.ceil(sampleCount * 0.99d));
+ long cumulative = 0L;
+ String p95 = "N/A";
+ String p99 = "N/A";
+ String max = "N/A";
+
+ for (int bucketIndex = 0; bucketIndex < currentBuckets.length; bucketIndex++) {
+ final long bucketCount =
+ currentBuckets[bucketIndex]
+ - (previousBuckets == null ? 0L : previousBuckets[bucketIndex]);
+ if (bucketCount <= 0) {
+ continue;
+ }
+
+ cumulative += bucketCount;
+ if ("N/A".equals(p95) && cumulative >= p95Threshold) {
+ p95 = LatencyHistogram.bucketLabel(bucketIndex);
+ }
+ if ("N/A".equals(p99) && cumulative >= p99Threshold) {
+ p99 = LatencyHistogram.bucketLabel(bucketIndex);
+ }
+ }
+
+ for (int bucketIndex = currentBuckets.length - 1; bucketIndex >= 0; bucketIndex--) {
+ final long bucketCount =
+ currentBuckets[bucketIndex]
+ - (previousBuckets == null ? 0L : previousBuckets[bucketIndex]);
+ if (bucketCount > 0) {
+ max = LatencyHistogram.bucketLabel(bucketIndex);
+ break;
+ }
+ }
+
+ return new LatencySummary(sampleCount, sumMs, p95, p99, max);
+ }
+
+ private static LatencySummary empty() {
+ return new LatencySummary(0L, 0L, "N/A", "N/A", "N/A");
+ }
+
+ private String formatAverageMs() {
+ if (sampleCount <= 0) {
+ return "N/A";
+ }
+ return String.format(Locale.ROOT, "%.2f", sumMs / (double) sampleCount);
+ }
+ }
+}
diff --git a/example/subscription/src/main/java/org/apache/iotdb/ConsensusSubscriptionTableTest.java b/example/subscription/src/main/java/org/apache/iotdb/ConsensusSubscriptionTableTest.java
new file mode 100644
index 0000000000000..a9cb090ccf8e0
--- /dev/null
+++ b/example/subscription/src/main/java/org/apache/iotdb/ConsensusSubscriptionTableTest.java
@@ -0,0 +1,2216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb;
+
+import org.apache.iotdb.isession.ITableSession;
+import org.apache.iotdb.rpc.subscription.config.TopicConstant;
+import org.apache.iotdb.rpc.subscription.payload.poll.RegionProgress;
+import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext;
+import org.apache.iotdb.rpc.subscription.payload.poll.TopicProgress;
+import org.apache.iotdb.rpc.subscription.payload.poll.WriterId;
+import org.apache.iotdb.rpc.subscription.payload.poll.WriterProgress;
+import org.apache.iotdb.session.TableSessionBuilder;
+import org.apache.iotdb.session.subscription.ISubscriptionTableSession;
+import org.apache.iotdb.session.subscription.SubscriptionTableSessionBuilder;
+import org.apache.iotdb.session.subscription.consumer.ISubscriptionTablePullConsumer;
+import org.apache.iotdb.session.subscription.consumer.base.ColumnAlignProcessor;
+import org.apache.iotdb.session.subscription.consumer.base.WatermarkProcessor;
+import org.apache.iotdb.session.subscription.consumer.table.SubscriptionTablePullConsumer;
+import org.apache.iotdb.session.subscription.consumer.table.SubscriptionTablePullConsumerBuilder;
+import org.apache.iotdb.session.subscription.payload.SubscriptionMessage;
+import org.apache.iotdb.session.subscription.payload.SubscriptionRecordHandler.SubscriptionResultSet;
+
+import org.apache.tsfile.enums.ColumnCategory;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
+/** TODO: Move these manual tests into ITs */
+public class ConsensusSubscriptionTableTest {
+
+ private static final String HOST = "127.0.0.1";
+ private static final int PORT = 6667;
+ private static final String USER = "root";
+ private static final String PASSWORD = "root";
+
+ private static int testCounter = 0;
+ private static int passed = 0;
+ private static int failed = 0;
+ private static final List failedTests = new ArrayList<>();
+
+ public static void main(String[] args) throws Exception {
+ System.out.println("=== Consensus-Based Subscription Table Model Test Suite ===\n");
+
+ String targetTest = args.length > 0 ? args[0] : null;
+
+ if (targetTest == null || "testBasicFlow".equals(targetTest)) {
+ runTest("testBasicFlow", ConsensusSubscriptionTableTest::testBasicFlow);
+ }
+ if (targetTest == null || "testDataTypes".equals(targetTest)) {
+ runTest("testDataTypes", ConsensusSubscriptionTableTest::testDataTypes);
+ }
+ if (targetTest == null || "testFilteringAndTopicSelection".equals(targetTest)) {
+ runTest(
+ "testFilteringAndTopicSelection",
+ ConsensusSubscriptionTableTest::testFilteringAndTopicSelection);
+ }
+ if (targetTest == null || "testSubscribeBeforeRegion".equals(targetTest)) {
+ runTest(
+ "testSubscribeBeforeRegion", ConsensusSubscriptionTableTest::testSubscribeBeforeRegion);
+ }
+ if (targetTest == null || "testMultiEntityIsolation".equals(targetTest)) {
+ runTest("testMultiEntityIsolation", ConsensusSubscriptionTableTest::testMultiEntityIsolation);
+ }
+ if (targetTest == null || "testWalCatchUpAndGapRecovery".equals(targetTest)) {
+ runTest(
+ "testWalCatchUpAndGapRecovery",
+ ConsensusSubscriptionTableTest::testWalCatchUpAndGapRecovery);
+ }
+ if (targetTest == null || "testSeekAndPositionSemantics".equals(targetTest)) {
+ runTest(
+ "testSeekAndPositionSemantics",
+ ConsensusSubscriptionTableTest::testSeekAndPositionSemantics);
+ }
+ if (targetTest == null || "testConsumerRestartRecovery".equals(targetTest)) {
+ runTest(
+ "testConsumerRestartRecovery",
+ ConsensusSubscriptionTableTest::testConsumerRestartRecovery);
+ }
+ if (targetTest == null || "testAckNackAndPoisonSemantics".equals(targetTest)) {
+ runTest(
+ "testAckNackAndPoisonSemantics",
+ ConsensusSubscriptionTableTest::testAckNackAndPoisonSemantics);
+ }
+ if (targetTest == null || "testProcessorWatermarkAndMetadata".equals(targetTest)) {
+ runTest(
+ "testProcessorWatermarkAndMetadata",
+ ConsensusSubscriptionTableTest::testProcessorWatermarkAndMetadata);
+ }
+
+ // Summary
+ System.out.println("\n=== Test Suite Summary ===");
+ System.out.println("Passed: " + passed);
+ System.out.println("Failed: " + failed);
+ if (!failedTests.isEmpty()) {
+ System.out.println("Failed tests: " + failedTests);
+ }
+ System.out.println("=== Done ===");
+ }
+
+ // ============================
+ // Test Infrastructure
+ // ============================
+
+ @FunctionalInterface
+ interface TestMethod {
+ void run() throws Exception;
+ }
+
+ private static void runTest(String name, TestMethod test) {
+ System.out.println("\n" + "=================================================================");
+ System.out.println("Running: " + name);
+ System.out.println("=================================================================");
+ try {
+ test.run();
+ passed++;
+ System.out.println(">>> PASSED: " + name);
+ } catch (AssertionError e) {
+ failed++;
+ failedTests.add(name);
+ System.out.println(">>> FAILED: " + name + " - " + e.getMessage());
+ e.printStackTrace(System.out);
+ } catch (Exception e) {
+ failed++;
+ failedTests.add(name);
+ System.out.println(">>> ERROR: " + name + " - " + e.getMessage());
+ e.printStackTrace(System.out);
+ }
+ }
+
+ private static String nextDatabase() {
+ testCounter++;
+ return "csub_tbl_" + testCounter;
+ }
+
+ private static String nextTopic() {
+ return "topic_tbl_" + testCounter;
+ }
+
+ private static String nextConsumerGroup() {
+ return "cg_tbl_" + testCounter;
+ }
+
+ private static String nextConsumerId() {
+ return "consumer_tbl_" + testCounter;
+ }
+
+ private static ITableSession openTableSession() throws Exception {
+ return new TableSessionBuilder()
+ .nodeUrls(Collections.singletonList(HOST + ":" + PORT))
+ .username(USER)
+ .password(PASSWORD)
+ .build();
+ }
+
+ private static void createDatabaseAndTable(
+ ITableSession session, String database, String tableName, String tableSchema)
+ throws Exception {
+ session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS " + database);
+ session.executeNonQueryStatement("USE " + database);
+ session.executeNonQueryStatement(String.format("CREATE TABLE %s (%s)", tableName, tableSchema));
+ }
+
+ private static void deleteDatabase(String database) {
+ try (ITableSession session = openTableSession()) {
+ session.executeNonQueryStatement("DROP DATABASE IF EXISTS " + database);
+ } catch (Exception e) {
+ // ignore
+ }
+ }
+
+ private static void dropTopicTable(String topicName) {
+ try (ISubscriptionTableSession subSession =
+ new SubscriptionTableSessionBuilder()
+ .host(HOST)
+ .port(PORT)
+ .username(USER)
+ .password(PASSWORD)
+ .build()) {
+ subSession.dropTopicIfExists(topicName);
+ } catch (Exception e) {
+ // ignore
+ }
+ }
+
+ private static void createTopicTable(String topicName, String dbKey, String tableKey)
+ throws Exception {
+ try (ISubscriptionTableSession subSession =
+ new SubscriptionTableSessionBuilder()
+ .host(HOST)
+ .port(PORT)
+ .username(USER)
+ .password(PASSWORD)
+ .build()) {
+ try {
+ subSession.dropTopicIfExists(topicName);
+ } catch (Exception e) {
+ // ignore
+ }
+
+ Properties topicConfig = new Properties();
+ topicConfig.put(TopicConstant.MODE_KEY, TopicConstant.MODE_LIVE_VALUE);
+ topicConfig.put(
+ TopicConstant.FORMAT_KEY, TopicConstant.FORMAT_SESSION_DATA_SETS_HANDLER_VALUE);
+ topicConfig.put(TopicConstant.DATABASE_KEY, dbKey);
+ topicConfig.put(TopicConstant.TABLE_KEY, tableKey);
+ subSession.createTopic(topicName, topicConfig);
+ System.out.println(
+ " Created topic: " + topicName + " (database=" + dbKey + ", table=" + tableKey + ")");
+ }
+ }
+
+ private static ISubscriptionTablePullConsumer createConsumer(
+ String consumerId, String consumerGroupId) throws Exception {
+ ISubscriptionTablePullConsumer consumer =
+ new SubscriptionTablePullConsumerBuilder()
+ .host(HOST)
+ .port(PORT)
+ .consumerId(consumerId)
+ .consumerGroupId(consumerGroupId)
+ .autoCommit(false)
+ .build();
+ consumer.open();
+ return consumer;
+ }
+
+ // ============================
+ // Polling & Verification
+ // ============================
+
+ /**
+ * Poll and commit messages. After reaching expectedRows, continues polling for 5 consecutive
+ * empty rounds to verify no extra data arrives.
+ */
+ private static PollResult pollUntilComplete(
+ ISubscriptionTablePullConsumer consumer, int expectedRows, int maxPollAttempts) {
+ return pollUntilComplete(consumer, expectedRows, maxPollAttempts, 1000, true);
+ }
+
+ /**
+ * Poll until we accumulate the expected number of rows, then verify no extra data arrives.
+ *
+ * After reaching expectedRows, continues polling until 5 consecutive empty polls confirm
+ * quiescence. Any extra rows polled are included in the count (will break assertEquals).
+ *
+ * @param commitMessages if false, messages are NOT committed
+ */
+ private static PollResult pollUntilComplete(
+ ISubscriptionTablePullConsumer consumer,
+ int expectedRows,
+ int maxPollAttempts,
+ long pollTimeoutMs,
+ boolean commitMessages) {
+ PollResult result = new PollResult();
+ int consecutiveEmpty = 0;
+
+ for (int attempt = 1; attempt <= maxPollAttempts; attempt++) {
+ List messages = consumer.poll(Duration.ofMillis(pollTimeoutMs));
+
+ if (messages.isEmpty()) {
+ consecutiveEmpty++;
+ // Normal completion: reached expected rows and verified quiescence
+ if (consecutiveEmpty >= 3 && result.totalRows >= expectedRows) {
+ System.out.println(
+ " Verified: "
+ + consecutiveEmpty
+ + " consecutive empty polls after "
+ + result.totalRows
+ + " rows (expected "
+ + expectedRows
+ + ")");
+ break;
+ }
+ // Stuck: have data but cannot reach expected count
+ if (consecutiveEmpty >= 5 && result.totalRows > 0) {
+ System.out.println(
+ " Stuck: "
+ + consecutiveEmpty
+ + " consecutive empty polls at "
+ + result.totalRows
+ + " rows (expected "
+ + expectedRows
+ + ")");
+ break;
+ }
+ // Never received anything
+ if (consecutiveEmpty >= 10 && result.totalRows == 0 && expectedRows > 0) {
+ System.out.println(" No data received after " + consecutiveEmpty + " polls");
+ break;
+ }
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException ignored) {
+ }
+ continue;
+ }
+
+ consecutiveEmpty = 0;
+
+ for (SubscriptionMessage message : messages) {
+ for (SubscriptionResultSet dataSet : getResultSets(message)) {
+ String tableName = dataSet.getTableName();
+ String databaseName = dataSet.getDatabaseName();
+ List columnNames = dataSet.getColumnNames();
+
+ while (dataSet.hasNext()) {
+ try {
+ org.apache.tsfile.read.common.RowRecord record = dataSet.nextRecord();
+ result.totalRows++;
+ if (tableName != null) {
+ result.rowsPerTable.merge(tableName, 1, Integer::sum);
+ }
+ if (databaseName != null) {
+ result.rowsPerDatabase.merge(databaseName, 1, Integer::sum);
+ }
+ for (int i = 0; i < columnNames.size(); i++) {
+ result.seenColumns.add(columnNames.get(i));
+ }
+ if (result.totalRows <= 5) {
+ System.out.println(
+ " Row: time="
+ + record.getTimestamp()
+ + ", values="
+ + record.getFields()
+ + ", table="
+ + tableName
+ + ", database="
+ + databaseName);
+ }
+ } catch (java.io.IOException e) {
+ throw new RuntimeException("Failed to iterate subscription result set", e);
+ }
+ }
+ }
+ if (commitMessages) {
+ consumer.commitSync(message);
+ }
+ }
+
+ System.out.println(
+ " Poll attempt "
+ + attempt
+ + ": totalRows="
+ + result.totalRows
+ + " / expected="
+ + expectedRows);
+
+ // Stop immediately if we exceeded the expected row count
+ if (expectedRows > 0 && result.totalRows > expectedRows) {
+ System.out.println(
+ " EXCEEDED: totalRows=" + result.totalRows + " > expectedRows=" + expectedRows);
+ break;
+ }
+ }
+
+ return result;
+ }
+
+ // ============================
+ // Cleanup
+ // ============================
+
+ /** Clean up all test artifacts: unsubscribe, close consumer, drop topic, delete database. */
+ private static void cleanup(
+ ISubscriptionTablePullConsumer consumer, String topicName, String database) {
+ if (consumer != null) {
+ try {
+ consumer.unsubscribe(topicName);
+ } catch (Exception e) {
+ // ignore
+ }
+ try {
+ consumer.close();
+ } catch (Exception e) {
+ // ignore
+ }
+ }
+ dropTopicTable(topicName);
+ deleteDatabase(database);
+ }
+
+ /** Clean up with multiple databases. */
+ private static void cleanup(
+ ISubscriptionTablePullConsumer consumer, String topicName, String... databases) {
+ if (consumer != null) {
+ try {
+ consumer.unsubscribe(topicName);
+ } catch (Exception e) {
+ // ignore
+ }
+ try {
+ consumer.close();
+ } catch (Exception e) {
+ // ignore
+ }
+ }
+ dropTopicTable(topicName);
+ for (String db : databases) {
+ deleteDatabase(db);
+ }
+ }
+
+ // ============================
+ // Result & Assertions
+ // ============================
+
+ static class PollResult {
+ int totalRows = 0;
+ Map rowsPerTable = new HashMap<>();
+ Map rowsPerDatabase = new HashMap<>();
+ Set seenColumns = new HashSet<>();
+
+ @Override
+ public String toString() {
+ return "PollResult{totalRows="
+ + totalRows
+ + ", rowsPerTable="
+ + rowsPerTable
+ + ", rowsPerDatabase="
+ + rowsPerDatabase
+ + ", seenColumns="
+ + seenColumns
+ + "}";
+ }
+ }
+
+ private static void assertEquals(String msg, int expected, int actual) {
+ if (expected != actual) {
+ throw new AssertionError(msg + ": expected=" + expected + ", actual=" + actual);
+ }
+ }
+
+ private static void assertEquals(String msg, long expected, long actual) {
+ if (expected != actual) {
+ throw new AssertionError(msg + ": expected=" + expected + ", actual=" + actual);
+ }
+ }
+
+ private static void assertEquals(String msg, String expected, String actual) {
+ if (expected == null ? actual != null : !expected.equals(actual)) {
+ throw new AssertionError(msg + ": expected=" + expected + ", actual=" + actual);
+ }
+ }
+
+ private static void assertTrue(String msg, boolean condition) {
+ if (!condition) {
+ throw new AssertionError(msg);
+ }
+ }
+
+ private static void assertAtLeast(String msg, int min, int actual) {
+ if (actual < min) {
+ throw new AssertionError(msg + ": expected at least " + min + ", actual=" + actual);
+ }
+ }
+
+ private static int countWriterFrontiers(TopicProgress topicProgress) {
+ int writerCount = 0;
+ if (topicProgress == null || topicProgress.getRegionProgress() == null) {
+ return 0;
+ }
+ for (Map.Entry entry : topicProgress.getRegionProgress().entrySet()) {
+ if (entry.getValue() != null && entry.getValue().getWriterPositions() != null) {
+ writerCount += entry.getValue().getWriterPositions().size();
+ }
+ }
+ return writerCount;
+ }
+
+ private static int countRows(SubscriptionMessage message) {
+ int rows = 0;
+ for (SubscriptionResultSet dataSet : getResultSets(message)) {
+ while (dataSet.hasNext()) {
+ try {
+ dataSet.next();
+ rows++;
+ } catch (java.io.IOException e) {
+ throw new RuntimeException("Failed to count rows from subscription result set", e);
+ }
+ }
+ }
+ return rows;
+ }
+
+ // ======================================================================
+ // High-signal 10-test suite wrappers
+ // ======================================================================
+
+ private static void testFilteringAndTopicSelection() throws Exception {
+ testPathFiltering();
+ testPollWithInfoTopicFilter();
+ }
+
+ private static void testWalCatchUpAndGapRecovery() throws Exception {
+ testBurstWriteGapRecovery();
+ }
+
+ private static void testSeekAndPositionSemantics() throws Exception {
+ testSeek();
+ }
+
+ private static void testAckNackAndPoisonSemantics() throws Exception {
+ testCommitAfterUnsubscribe();
+ testPoisonMessageDrop();
+ }
+
+ private static void testProcessorWatermarkAndMetadata() throws Exception {
+ testProcessorFramework();
+ testWriterProgressFields();
+ }
+
+ // ======================================================================
+ // Topic filter subcase for table model
+ // ======================================================================
+ private static void testPollWithInfoTopicFilter() throws Exception {
+ String database = nextDatabase();
+ String topicName1 = "topic_tbl_filter_" + testCounter + "_a";
+ String topicName2 = "topic_tbl_filter_" + testCounter + "_b";
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId = nextConsumerId();
+ SubscriptionTablePullConsumer consumer = null;
+
+ try {
+ try (ITableSession session = openTableSession()) {
+ createDatabaseAndTable(session, database, "t1", "tag1 STRING TAG, s1 INT64 FIELD");
+ session.executeNonQueryStatement("USE " + database);
+ session.executeNonQueryStatement("CREATE TABLE t2 (tag1 STRING TAG, s1 INT64 FIELD)");
+ }
+ Thread.sleep(2000);
+
+ createTopicTable(topicName1, database, "t1");
+ createTopicTable(topicName2, database, "t2");
+ Thread.sleep(1000);
+
+ consumer = (SubscriptionTablePullConsumer) createConsumer(consumerId, consumerGroupId);
+ consumer.subscribe(topicName1, topicName2);
+ Thread.sleep(3000);
+
+ try (ITableSession session = openTableSession()) {
+ session.executeNonQueryStatement("USE " + database);
+ for (int i = 1; i <= 40; i++) {
+ if (i <= 30) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO t1 (tag1, s1, time) VALUES ('d1', %d, %d)", i * 10, i));
+ }
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO t2 (tag1, s1, time) VALUES ('d2', %d, %d)", i * 20, i));
+ }
+ }
+ Thread.sleep(3000);
+
+ int t1Rows = 0;
+ Set topic1Only = new HashSet<>(Arrays.asList(topicName1));
+ for (int attempt = 0; attempt < 40; attempt++) {
+ org.apache.iotdb.session.subscription.payload.PollResult pollResult =
+ consumer.pollWithInfo(topic1Only, 2000);
+ List msgs = pollResult.getMessages();
+ if (msgs.isEmpty()) {
+ if (t1Rows > 0) {
+ break;
+ }
+ Thread.sleep(1000);
+ continue;
+ }
+ for (SubscriptionMessage msg : msgs) {
+ for (SubscriptionResultSet ds : getResultSets(msg)) {
+ while (ds.hasNext()) {
+ ds.next();
+ t1Rows++;
+ assertEquals("Topic1-only poll should stay on t1", "t1", ds.getTableName());
+ }
+ }
+ consumer.commitSync(msg);
+ }
+ }
+ assertEquals("Topic1 should deliver exactly 30 rows from t1", 30, t1Rows);
+
+ int t2Rows = 0;
+ Set topic2Only = new HashSet<>(Arrays.asList(topicName2));
+ for (int attempt = 0; attempt < 40; attempt++) {
+ org.apache.iotdb.session.subscription.payload.PollResult pollResult =
+ consumer.pollWithInfo(topic2Only, 2000);
+ List msgs = pollResult.getMessages();
+ if (msgs.isEmpty()) {
+ if (t2Rows > 0) {
+ break;
+ }
+ Thread.sleep(1000);
+ continue;
+ }
+ for (SubscriptionMessage msg : msgs) {
+ for (SubscriptionResultSet ds : getResultSets(msg)) {
+ while (ds.hasNext()) {
+ ds.next();
+ t2Rows++;
+ assertEquals("Topic2-only poll should stay on t2", "t2", ds.getTableName());
+ }
+ }
+ consumer.commitSync(msg);
+ }
+ }
+ assertEquals("Topic2 should deliver exactly 40 rows from t2", 40, t2Rows);
+ } finally {
+ if (consumer != null) {
+ try {
+ consumer.unsubscribe(topicName1, topicName2);
+ } catch (Exception e) {
+ // ignore
+ }
+ try {
+ consumer.close();
+ } catch (Exception e) {
+ // ignore
+ }
+ }
+ dropTopicTable(topicName1);
+ dropTopicTable(topicName2);
+ deleteDatabase(database);
+ }
+ }
+
+ // ======================================================================
+ // Test 8: Consumer Restart Recovery
+ // ======================================================================
+ private static void testConsumerRestartRecovery() throws Exception {
+ String database = nextDatabase();
+ String topicName = nextTopic();
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId1 = nextConsumerId();
+ String consumerId2 = consumerId1 + "_restart";
+ SubscriptionTablePullConsumer consumer1 = null;
+ SubscriptionTablePullConsumer consumer2 = null;
+
+ try {
+ try (ITableSession session = openTableSession()) {
+ createDatabaseAndTable(session, database, "t1", "tag1 STRING TAG, s1 INT64 FIELD");
+ }
+ Thread.sleep(2000);
+
+ createTopicTable(topicName, database, "t1");
+ Thread.sleep(1000);
+
+ consumer1 = (SubscriptionTablePullConsumer) createConsumer(consumerId1, consumerGroupId);
+ consumer1.subscribe(topicName);
+ Thread.sleep(3000);
+
+ final int totalRows = 257;
+ try (ITableSession session = openTableSession()) {
+ session.executeNonQueryStatement("USE " + database);
+ for (int i = 1; i <= totalRows; i++) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO t1 (tag1, s1, time) VALUES ('d1', %d, %d)", i * 10, i));
+ }
+ session.executeNonQueryStatement("FLUSH");
+ }
+ Thread.sleep(3000);
+
+ int committedRows = 0;
+ for (int attempt = 1; attempt <= 30; attempt++) {
+ List messages = consumer1.poll(Duration.ofMillis(2000));
+ if (messages.isEmpty()) {
+ Thread.sleep(1000);
+ continue;
+ }
+ SubscriptionMessage firstMessage = messages.get(0);
+ committedRows = countRows(firstMessage);
+ consumer1.commitSync(firstMessage);
+ break;
+ }
+
+ assertAtLeast("First consumer should commit some rows before restart", 1, committedRows);
+ TopicProgress checkpoint = consumer1.committedPositions(topicName);
+ assertTrue(
+ "Committed checkpoint should not be empty",
+ checkpoint.getRegionProgress() != null && !checkpoint.getRegionProgress().isEmpty());
+ int remainingRows = totalRows - committedRows;
+ assertAtLeast("Restart scenario should leave rows after the first commit", 1, remainingRows);
+
+ consumer1.close();
+ consumer1 = null;
+
+ consumer2 = (SubscriptionTablePullConsumer) createConsumer(consumerId2, consumerGroupId);
+ consumer2.subscribe(topicName);
+ Thread.sleep(3000);
+ consumer2.seekAfter(topicName, checkpoint);
+ Thread.sleep(1000);
+
+ PollResult resumed = pollUntilComplete(consumer2, remainingRows, 120);
+ assertEquals(
+ "Restarted consumer should resume from the committed checkpoint without replay",
+ remainingRows,
+ resumed.totalRows);
+ } finally {
+ cleanup(consumer1, topicName, database);
+ cleanup(consumer2, topicName, database);
+ }
+ }
+
+ // ======================================================================
+ // Test 1: Basic Flow (merged: BasicDataDelivery + MultiTables + Flush)
+ // ======================================================================
+ /**
+ * Verifies:
+ *
+ *
+ * - Data written BEFORE subscribe is NOT received
+ *
- Multiple tables (t1, t2, t3) written AFTER subscribe are all received
+ *
- Flush does not cause data loss (WAL pinning keeps entries available)
+ *
- Exact row count matches expectation
+ *
+ */
+ private static void testBasicFlow() throws Exception {
+ String database = nextDatabase();
+ String topicName = nextTopic();
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId = nextConsumerId();
+ ISubscriptionTablePullConsumer consumer = null;
+
+ try {
+ // Step 1: Write initial data to create DataRegion (should NOT be received)
+ System.out.println(" Step 1: Writing initial data (should NOT be received)");
+ try (ITableSession session = openTableSession()) {
+ createDatabaseAndTable(session, database, "t1", "tag1 STRING TAG, s1 INT64 FIELD");
+ session.executeNonQueryStatement("USE " + database);
+ session.executeNonQueryStatement("CREATE TABLE t2 (tag1 STRING TAG, s1 INT64 FIELD)");
+ session.executeNonQueryStatement("CREATE TABLE t3 (tag1 STRING TAG, s1 INT64 FIELD)");
+ for (int i = 0; i < 50; i++) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO t1 (tag1, s1, time) VALUES ('d1', %d, %d)", i * 10, i));
+ }
+ session.executeNonQueryStatement("INSERT INTO t2 (tag1, s1, time) VALUES ('d1', 0, 0)");
+ session.executeNonQueryStatement("INSERT INTO t3 (tag1, s1, time) VALUES ('d1', 0, 0)");
+ session.executeNonQueryStatement("flush");
+ }
+ Thread.sleep(2000);
+
+ // Step 2: Create topic and subscribe
+ System.out.println(" Step 2: Creating topic and subscribing");
+ createTopicTable(topicName, database, ".*");
+ Thread.sleep(1000);
+
+ consumer = createConsumer(consumerId, consumerGroupId);
+ consumer.subscribe(topicName);
+ Thread.sleep(3000);
+
+ // Step 3: Write to 3 tables (30 rows each = 90 total), then flush
+ System.out.println(" Step 3: Writing 30 rows x 3 tables AFTER subscribe, then flush");
+ try (ITableSession session = openTableSession()) {
+ session.executeNonQueryStatement("USE " + database);
+ for (int i = 100; i < 130; i++) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO t1 (tag1, s1, time) VALUES ('d1', %d, %d)", i * 10, i));
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO t2 (tag1, s1, time) VALUES ('d1', %d, %d)", i * 20, i));
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO t3 (tag1, s1, time) VALUES ('d1', %d, %d)", i * 30, i));
+ }
+ System.out.println(" Flushing...");
+ session.executeNonQueryStatement("flush");
+ }
+ Thread.sleep(2000);
+
+ // Step 4: Poll and verify
+ System.out.println(" Step 4: Polling...");
+ PollResult result = pollUntilComplete(consumer, 90, 100);
+ System.out.println(" Result: " + result);
+
+ assertEquals("Expected exactly 90 rows (30 per table)", 90, result.totalRows);
+ if (!result.rowsPerTable.isEmpty()) {
+ System.out.println(" Rows per table: " + result.rowsPerTable);
+ for (String tbl : new String[] {"t1", "t2", "t3"}) {
+ Integer tblRows = result.rowsPerTable.get(tbl);
+ assertAtLeast("Expected rows from " + tbl, 1, tblRows != null ? tblRows : 0);
+ }
+ }
+ } finally {
+ cleanup(consumer, topicName, database);
+ }
+ }
+
+ // ======================================================================
+ // Test 2: Data Types (merged: MultipleDataTypes + MultiColumnTypes + CrossPartition)
+ // ======================================================================
+ /**
+ * Verifies:
+ *
+ *
+ * - Non-aligned: 6 data types via separate INSERTs
+ *
- All-column: 6 fields in a single INSERT
+ *
- Cross-partition: timestamps >1 week apart via SQL, Tablet methods
+ *
+ */
+ private static void testDataTypes() throws Exception {
+ String database = nextDatabase();
+ String topicName = nextTopic();
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId = nextConsumerId();
+ ISubscriptionTablePullConsumer consumer = null;
+ final long GAP = 604_800_001L; // slightly over 1 week
+
+ try {
+ try (ITableSession session = openTableSession()) {
+ createDatabaseAndTable(
+ session,
+ database,
+ "t1",
+ "tag1 STRING TAG, s_int32 INT32 FIELD, s_int64 INT64 FIELD, "
+ + "s_float FLOAT FIELD, s_double DOUBLE FIELD, s_bool BOOLEAN FIELD, "
+ + "s_text TEXT FIELD");
+ session.executeNonQueryStatement("USE " + database);
+ // Init row to force DataRegion creation
+ session.executeNonQueryStatement(
+ "INSERT INTO t1 (tag1, s_int32, s_int64, s_float, s_double, s_bool, s_text, time) "
+ + "VALUES ('d1', 0, 0, 0.0, 0.0, false, 'init', 0)");
+ session.executeNonQueryStatement("flush");
+ }
+ Thread.sleep(2000);
+
+ createTopicTable(topicName, database, ".*");
+ Thread.sleep(1000);
+
+ consumer = createConsumer(consumerId, consumerGroupId);
+ consumer.subscribe(topicName);
+ Thread.sleep(3000);
+
+ int totalExpected = 0;
+ try (ITableSession session = openTableSession()) {
+ session.executeNonQueryStatement("USE " + database);
+
+ // --- Part A: 6 data types x 20 rows, separate INSERTs ---
+ System.out.println(" Part A: 6 data types x 20 rows (separate INSERTs)");
+ for (int i = 1; i <= 20; i++) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO t1 (tag1, s_int32, time) VALUES ('d1', %d, %d)", i, i));
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO t1 (tag1, s_int64, time) VALUES ('d1', %d, %d)",
+ (long) i * 100000L, i));
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO t1 (tag1, s_float, time) VALUES ('d1', %f, %d)", i * 1.1f, i));
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO t1 (tag1, s_double, time) VALUES ('d1', %f, %d)", i * 2.2, i));
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO t1 (tag1, s_bool, time) VALUES ('d1', %s, %d)",
+ i % 2 == 0 ? "true" : "false", i));
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO t1 (tag1, s_text, time) VALUES ('d1', 'text_%d', %d)", i, i));
+ }
+ totalExpected += 120; // 6 types x 20 rows
+
+ // --- Part B: All-column rows (50 rows) ---
+ System.out.println(" Part B: 50 all-column rows");
+ for (int i = 21; i <= 70; i++) {
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO t1 (tag1, s_int32, s_int64, s_float, s_double, s_bool, s_text, time)"
+ + " VALUES ('d1', %d, %d, %f, %f, %s, 'text_%d', %d)",
+ i, (long) i * 100000L, i * 1.1f, i * 2.2, i % 2 == 0 ? "true" : "false", i, i));
+ }
+ totalExpected += 50;
+
+ // --- Part C: Cross-partition writes ---
+ System.out.println(" Part C: Cross-partition (SQL single, multi, Tablet)");
+ long baseTs = 1_000_000_000L;
+
+ // SQL single-row x2
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO t1 (tag1, s_int32, s_int64, s_float, s_double, s_bool, s_text, time) "
+ + "VALUES ('d1', 1, 100, 1.1, 1.11, true, 'xp_single_1', %d)",
+ baseTs));
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO t1 (tag1, s_int32, s_int64, s_float, s_double, s_bool, s_text, time) "
+ + "VALUES ('d1', 2, 200, 2.2, 2.22, false, 'xp_single_2', %d)",
+ baseTs + GAP));
+ totalExpected += 2;
+
+ // SQL multi-row x3
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO t1 (tag1, s_int32, s_int64, s_float, s_double, s_bool, s_text, time) "
+ + "VALUES ('d1', 3, 300, 3.3, 3.33, true, 'xp_multi_1', %d), "
+ + "('d1', 4, 400, 4.4, 4.44, false, 'xp_multi_2', %d), "
+ + "('d1', 5, 500, 5.5, 5.55, true, 'xp_multi_3', %d)",
+ baseTs + GAP * 2, baseTs + GAP * 3, baseTs + GAP * 4));
+ totalExpected += 3;
+
+ // Tablet x4
+ List schemaList = new ArrayList<>();
+ schemaList.add(new MeasurementSchema("tag1", TSDataType.STRING));
+ schemaList.add(new MeasurementSchema("s_int32", TSDataType.INT32));
+ schemaList.add(new MeasurementSchema("s_int64", TSDataType.INT64));
+ schemaList.add(new MeasurementSchema("s_float", TSDataType.FLOAT));
+ schemaList.add(new MeasurementSchema("s_double", TSDataType.DOUBLE));
+ schemaList.add(new MeasurementSchema("s_bool", TSDataType.BOOLEAN));
+ schemaList.add(new MeasurementSchema("s_text", TSDataType.STRING));
+
+ List categories =
+ java.util.Arrays.asList(
+ ColumnCategory.TAG,
+ ColumnCategory.FIELD,
+ ColumnCategory.FIELD,
+ ColumnCategory.FIELD,
+ ColumnCategory.FIELD,
+ ColumnCategory.FIELD,
+ ColumnCategory.FIELD);
+
+ Tablet tablet =
+ new Tablet(
+ "t1",
+ IMeasurementSchema.getMeasurementNameList(schemaList),
+ IMeasurementSchema.getDataTypeList(schemaList),
+ categories,
+ 10);
+ for (int i = 0; i < 4; i++) {
+ int row = tablet.getRowSize();
+ long ts = baseTs + GAP * (5 + i);
+ tablet.addTimestamp(row, ts);
+ tablet.addValue("tag1", row, "d1");
+ tablet.addValue("s_int32", row, 6 + i);
+ tablet.addValue("s_int64", row, (long) (600 + i * 100));
+ tablet.addValue("s_float", row, (6 + i) * 1.1f);
+ tablet.addValue("s_double", row, (6 + i) * 2.22);
+ tablet.addValue("s_bool", row, i % 2 == 0);
+ tablet.addValue("s_text", row, "xp_tablet_" + (i + 1));
+ }
+ session.insert(tablet);
+ totalExpected += 4;
+ }
+
+ System.out.println(" Total expected rows: " + totalExpected);
+ Thread.sleep(2000);
+
+ PollResult result = pollUntilComplete(consumer, totalExpected, 200);
+ System.out.println(" Result: " + result);
+
+ assertAtLeast(
+ "Expected at least " + totalExpected + " rows", totalExpected, result.totalRows);
+ assertAtLeast("Expected multiple column types in result", 2, result.seenColumns.size());
+ } finally {
+ cleanup(consumer, topicName, database);
+ }
+ }
+
+ // ======================================================================
+ // Test 3: Path Filtering (merged: TableLevel + DatabaseLevel)
+ // ======================================================================
+ /**
+ * Verifies:
+ *
+ *
+ * - Table-level: topic on table=t1 does NOT deliver t2 data
+ *
- Database-level: topic on db1 does NOT deliver db2 data
+ *
+ */
+ private static void testPathFiltering() throws Exception {
+ String database1 = nextDatabase();
+ String database2 = database1 + "_other";
+ String topicName = nextTopic();
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId = nextConsumerId();
+ ISubscriptionTablePullConsumer consumer = null;
+
+ try {
+ try (ITableSession session = openTableSession()) {
+ // db1 with t1 and t2
+ createDatabaseAndTable(session, database1, "t1", "tag1 STRING TAG, s1 INT64 FIELD");
+ session.executeNonQueryStatement("USE " + database1);
+ session.executeNonQueryStatement("CREATE TABLE t2 (tag1 STRING TAG, s1 INT64 FIELD)");
+ session.executeNonQueryStatement("INSERT INTO t1 (tag1, s1, time) VALUES ('d1', 0, 0)");
+ session.executeNonQueryStatement("INSERT INTO t2 (tag1, s1, time) VALUES ('d1', 0, 0)");
+ // db2 with t1
+ createDatabaseAndTable(session, database2, "t1", "tag1 STRING TAG, s1 INT64 FIELD");
+ session.executeNonQueryStatement("USE " + database2);
+ session.executeNonQueryStatement("INSERT INTO t1 (tag1, s1, time) VALUES ('d1', 0, 0)");
+ session.executeNonQueryStatement("flush");
+ }
+ Thread.sleep(2000);
+
+ // Topic: only db1, only table t1
+ createTopicTable(topicName, database1, "t1");
+ Thread.sleep(1000);
+
+ consumer = createConsumer(consumerId, consumerGroupId);
+ consumer.subscribe(topicName);
+ Thread.sleep(3000);
+
+ System.out.println(" Writing to db1.t1, db1.t2, db2.t1 (topic filter: db1.t1 only)");
+ try (ITableSession session = openTableSession()) {
+ session.executeNonQueryStatement("USE " + database1);
+ for (int i = 100; i < 150; i++) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO t1 (tag1, s1, time) VALUES ('d1', %d, %d)", i * 10, i));
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO t2 (tag1, s1, time) VALUES ('d1', %d, %d)", i * 20, i));
+ }
+ session.executeNonQueryStatement("USE " + database2);
+ for (int i = 100; i < 150; i++) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO t1 (tag1, s1, time) VALUES ('d1', %d, %d)", i * 30, i));
+ }
+ }
+ Thread.sleep(2000);
+
+ System.out.println(" Polling (expecting only db1.t1 data = 50 rows)...");
+ PollResult result = pollUntilComplete(consumer, 50, 60);
+ System.out.println(" Result: " + result);
+
+ assertEquals("Expected exactly 50 rows from db1.t1 only", 50, result.totalRows);
+ if (!result.rowsPerTable.isEmpty()) {
+ Integer t2Rows = result.rowsPerTable.get("t2");
+ assertTrue("Expected NO rows from t2, but got " + t2Rows, t2Rows == null || t2Rows == 0);
+ System.out.println(" Table filtering verified: t1 only");
+ }
+ if (!result.rowsPerDatabase.isEmpty()) {
+ Integer db2Rows = result.rowsPerDatabase.get(database2);
+ assertTrue("Expected NO rows from " + database2, db2Rows == null || db2Rows == 0);
+ System.out.println(" Database filtering verified: " + database1 + " only");
+ }
+ } finally {
+ cleanup(consumer, topicName, database1, database2);
+ }
+ }
+
+ // ======================================================================
+ // Test 4: Subscribe Before Region Creation (kept as-is)
+ // ======================================================================
+ /**
+ * Subscribe BEFORE the database/region exists, then create database and write. Tests the
+ * IoTConsensus.onNewPeerCreated auto-binding path with table model.
+ */
+ private static void testSubscribeBeforeRegion() throws Exception {
+ String database = nextDatabase();
+ String topicName = nextTopic();
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId = nextConsumerId();
+ ISubscriptionTablePullConsumer consumer = null;
+
+ try {
+ System.out.println(" Step 1: Creating topic BEFORE database exists");
+ createTopicTable(topicName, database, ".*");
+ Thread.sleep(1000);
+
+ System.out.println(" Step 2: Subscribing (no DataRegion exists yet)");
+ consumer = createConsumer(consumerId, consumerGroupId);
+ consumer.subscribe(topicName);
+ Thread.sleep(3000);
+
+ System.out.println(" Step 3: Creating database, table and writing data (100 rows)");
+ try (ITableSession session = openTableSession()) {
+ createDatabaseAndTable(session, database, "t1", "tag1 STRING TAG, s1 INT64 FIELD");
+ session.executeNonQueryStatement("USE " + database);
+ for (int i = 0; i < 100; i++) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO t1 (tag1, s1, time) VALUES ('d1', %d, %d)", i * 10, i));
+ }
+ }
+ Thread.sleep(5000);
+
+ System.out.println(" Step 4: Polling...");
+ PollResult result = pollUntilComplete(consumer, 100, 100);
+ System.out.println(" Result: " + result);
+
+ if (result.totalRows >= 100) {
+ System.out.println(" Auto-binding works! All " + result.totalRows + " rows received.");
+ } else if (result.totalRows > 0) {
+ System.out.println(
+ " Partial: " + result.totalRows + "/100 rows. First writes may precede binding.");
+ } else {
+ System.out.println(" No data received. Check logs for auto-binding messages.");
+ }
+ assertAtLeast(
+ "Expected some rows from subscribe-before-region (auto-binding)", 1, result.totalRows);
+ } finally {
+ cleanup(consumer, topicName, database);
+ }
+ }
+
+ // testRedelivery removed — will be re-added with proper timeout-based nack testing
+
+ // ======================================================================
+ // Test 6: Multi-Entity Isolation
+ // ======================================================================
+ /**
+ * Verifies:
+ *
+ *
+ * - Two consumer groups on the same topic each receive the full data stream independently
+ *
+ */
+ private static void testMultiEntityIsolation() throws Exception {
+ String database = nextDatabase();
+ String topicName = "topic_tbl_multi_" + testCounter;
+ String consumerGroupId1 = "cg_tbl_multi_" + testCounter + "_a";
+ String consumerId1 = "consumer_tbl_multi_" + testCounter + "_a";
+ String consumerGroupId2 = "cg_tbl_multi_" + testCounter + "_b";
+ String consumerId2 = "consumer_tbl_multi_" + testCounter + "_b";
+ ISubscriptionTablePullConsumer consumer1 = null;
+ ISubscriptionTablePullConsumer consumer2 = null;
+
+ try {
+ // Setup: database with a single table to isolate the multi-group semantics.
+ try (ITableSession session = openTableSession()) {
+ createDatabaseAndTable(session, database, "t1", "tag1 STRING TAG, s1 INT64 FIELD");
+ session.executeNonQueryStatement("USE " + database);
+ session.executeNonQueryStatement("INSERT INTO t1 (tag1, s1, time) VALUES ('d1', 0, 0)");
+ session.executeNonQueryStatement("flush");
+ }
+ Thread.sleep(2000);
+
+ createTopicTable(topicName, database, "t1");
+ Thread.sleep(1000);
+
+ consumer1 = createConsumer(consumerId1, consumerGroupId1);
+ consumer1.subscribe(topicName);
+ consumer2 = createConsumer(consumerId2, consumerGroupId2);
+ consumer2.subscribe(topicName);
+ Thread.sleep(3000);
+
+ System.out.println(" Writing 70 rows to t1");
+ try (ITableSession session = openTableSession()) {
+ session.executeNonQueryStatement("USE " + database);
+ for (int i = 1; i <= 70; i++) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO t1 (tag1, s1, time) VALUES ('d1', %d, %d)", i * 10, i));
+ }
+ }
+ Thread.sleep(2000);
+
+ System.out.println(" Multi-group isolation");
+ System.out.println(" Polling from group 1...");
+ PollResult result1 = pollUntilComplete(consumer1, 70, 80);
+ System.out.println(" Group 1 result: " + result1);
+
+ System.out.println(" Polling from group 2...");
+ PollResult result2 = pollUntilComplete(consumer2, 70, 80);
+ System.out.println(" Group 2 result: " + result2);
+
+ assertEquals("Group 1 should receive all 70 rows", 70, result1.totalRows);
+ assertEquals("Group 2 should receive all 70 rows", 70, result2.totalRows);
+ assertEquals("Expected 70 rows from t1", 70, result1.rowsPerTable.getOrDefault("t1", 0));
+ assertEquals("Expected 70 rows from t1", 70, result2.rowsPerTable.getOrDefault("t1", 0));
+ System.out.println(
+ " Multi-group isolation verified: group1="
+ + result1.totalRows
+ + ", group2="
+ + result2.totalRows);
+ } finally {
+ if (consumer1 != null) {
+ try {
+ consumer1.unsubscribe(topicName);
+ } catch (Exception e) {
+ /* ignore */
+ }
+ try {
+ consumer1.close();
+ } catch (Exception e) {
+ /* ignore */
+ }
+ }
+ if (consumer2 != null) {
+ try {
+ consumer2.unsubscribe(topicName);
+ } catch (Exception e) {
+ /* ignore */
+ }
+ try {
+ consumer2.close();
+ } catch (Exception e) {
+ /* ignore */
+ }
+ }
+ dropTopicTable(topicName);
+ deleteDatabase(database);
+ }
+ }
+
+ // ======================================================================
+ // Test 7: Burst Write Gap Recovery (NEW — tests C2 fix)
+ // ======================================================================
+ /**
+ * Tests that burst writing beyond the pending queue capacity (4096) does not cause data loss. The
+ * pending queue overflow triggers gaps, which should be recovered from WAL.
+ *
+ * Mechanism: Each {@code IoTConsensusServerImpl.write()} call produces exactly one
+ * {@code pendingEntries.offer()}. A single {@code session.insert(tablet)} with N rows in one time
+ * partition = 1 write() call = 1 offer, so Tablet batches rarely overflow the queue. To actually
+ * overflow, we need 4096+ individual write() calls arriving faster than the prefetch
+ * thread can drain. We achieve this with multiple concurrent writer threads, each performing
+ * individual SQL INSERTs, to maximize the aggregate write rate vs. drain rate.
+ *
+ *
Note: Gap occurrence is inherently timing-dependent (race between writers and the
+ * prefetch drain loop). This test maximizes the probability by using concurrent threads, but
+ * cannot guarantee gap occurrence on every run. Check server logs for "gap detected" / "Filling
+ * from WAL" messages to confirm the gap path was exercised.
+ *
+ *
Fix verified: C2 — gap entries are not skipped when WAL fill times out; they are deferred to
+ * the next prefetch iteration.
+ */
+ private static void testBurstWriteGapRecovery() throws Exception {
+ String database = nextDatabase();
+ String topicName = nextTopic();
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId = nextConsumerId();
+ ISubscriptionTablePullConsumer consumer = null;
+
+ try {
+ try (ITableSession session = openTableSession()) {
+ createDatabaseAndTable(session, database, "t1", "tag1 STRING TAG, s1 INT64 FIELD");
+ session.executeNonQueryStatement("USE " + database);
+ session.executeNonQueryStatement("INSERT INTO t1 (tag1, s1, time) VALUES ('d1', 0, 0)");
+ session.executeNonQueryStatement("flush");
+ }
+ Thread.sleep(2000);
+
+ createTopicTable(topicName, database, ".*");
+ Thread.sleep(1000);
+
+ consumer = createConsumer(consumerId, consumerGroupId);
+ consumer.subscribe(topicName);
+ Thread.sleep(3000);
+
+ // Use multiple concurrent writer threads with individual SQL INSERTs.
+ // Each INSERT → 1 IoTConsensusServerImpl.write() → 1 pendingEntries.offer().
+ // With N threads writing concurrently, aggregate rate should exceed drain rate
+ // and overflow the 4096-capacity queue, creating gaps.
+ final int writerThreads = 4;
+ final int rowsPerThread = 1500; // 4 * 1500 = 6000 total write() calls > 4096
+ final int totalRows = writerThreads * rowsPerThread;
+ final AtomicInteger errorCount = new AtomicInteger(0);
+ final CountDownLatch startLatch = new CountDownLatch(1);
+ final CountDownLatch doneLatch = new CountDownLatch(writerThreads);
+
+ System.out.println(
+ " Burst writing "
+ + totalRows
+ + " rows via "
+ + writerThreads
+ + " concurrent threads ("
+ + rowsPerThread
+ + " individual SQL INSERTs each)");
+ System.out.println(
+ " (Each INSERT = 1 WAL entry = 1 pendingEntries.offer(); " + "queue capacity = 4096)");
+
+ ExecutorService executor = Executors.newFixedThreadPool(writerThreads);
+ for (int t = 0; t < writerThreads; t++) {
+ final int threadId = t;
+ final int startTs = threadId * rowsPerThread + 1;
+ executor.submit(
+ () -> {
+ try {
+ startLatch.await(); // all threads start at the same time
+ try (ITableSession session = openTableSession()) {
+ session.executeNonQueryStatement("USE " + database);
+ for (int i = 0; i < rowsPerThread; i++) {
+ int ts = startTs + i;
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO t1 (tag1, s1, time) VALUES ('d1', %d, %d)",
+ (long) ts * 10, ts));
+ }
+ }
+ } catch (Exception e) {
+ System.out.println(" Writer thread " + threadId + " error: " + e.getMessage());
+ errorCount.incrementAndGet();
+ } finally {
+ doneLatch.countDown();
+ }
+ });
+ }
+
+ // Fire all threads simultaneously
+ startLatch.countDown();
+ doneLatch.await();
+ executor.shutdown();
+
+ if (errorCount.get() > 0) {
+ System.out.println(" WARNING: " + errorCount.get() + " writer threads encountered errors");
+ }
+
+ // Do NOT add artificial delay — let the consumer compete with ongoing WAL writes
+ System.out.println(
+ " Polling (expecting " + totalRows + " rows, may need WAL gap recovery)...");
+ System.out.println(
+ " (Check server logs for 'gap detected' to confirm gap recovery was triggered)");
+ PollResult result = pollUntilComplete(consumer, totalRows, 6000, 2000, true);
+ System.out.println(" Result: " + result);
+
+ assertEquals(
+ "Expected exactly " + totalRows + " rows (no data loss despite pending queue overflow)",
+ totalRows,
+ result.totalRows);
+ } finally {
+ cleanup(consumer, topicName, database);
+ }
+ }
+
+ // ======================================================================
+ // Test 8: Commit After Unsubscribe (NEW — tests H7 fix)
+ // ======================================================================
+ /**
+ * Tests that commit still works correctly after the consumer has unsubscribed (queue has been
+ * torn down). The commit routing should use metadata-based topic config check instead of runtime
+ * queue state.
+ *
+ *
Fix verified: H7 — commit routes via isConsensusBasedTopic() instead of hasQueue().
+ */
+ private static void testCommitAfterUnsubscribe() throws Exception {
+ String database = nextDatabase();
+ String topicName = nextTopic();
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId = nextConsumerId();
+ ISubscriptionTablePullConsumer consumer = null;
+
+ try {
+ try (ITableSession session = openTableSession()) {
+ createDatabaseAndTable(session, database, "t1", "tag1 STRING TAG, s1 INT64 FIELD");
+ session.executeNonQueryStatement("USE " + database);
+ session.executeNonQueryStatement("INSERT INTO t1 (tag1, s1, time) VALUES ('d1', 0, 0)");
+ session.executeNonQueryStatement("flush");
+ }
+ Thread.sleep(2000);
+
+ createTopicTable(topicName, database, ".*");
+ Thread.sleep(1000);
+
+ consumer = createConsumer(consumerId, consumerGroupId);
+ consumer.subscribe(topicName);
+ Thread.sleep(3000);
+
+ // Write data
+ System.out.println(" Writing 50 rows");
+ try (ITableSession session = openTableSession()) {
+ session.executeNonQueryStatement("USE " + database);
+ for (int i = 1; i <= 50; i++) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO t1 (tag1, s1, time) VALUES ('d1', %d, %d)", i * 10, i));
+ }
+ }
+ Thread.sleep(2000);
+
+ // Poll WITHOUT commit
+ System.out.println(" Polling WITHOUT commit...");
+ List uncommittedMessages = new ArrayList<>();
+ int polledRows = 0;
+ for (int attempt = 0; attempt < 60 && polledRows < 50; attempt++) {
+ List msgs = consumer.poll(Duration.ofMillis(2000));
+ if (msgs.isEmpty()) {
+ if (polledRows > 0) break;
+ Thread.sleep(500);
+ continue;
+ }
+ for (SubscriptionMessage msg : msgs) {
+ uncommittedMessages.add(msg);
+ for (SubscriptionResultSet ds : getResultSets(msg)) {
+ while (ds.hasNext()) {
+ ds.next();
+ polledRows++;
+ }
+ }
+ }
+ }
+ System.out.println(
+ " Polled "
+ + polledRows
+ + " rows, holding "
+ + uncommittedMessages.size()
+ + " uncommitted messages");
+ assertAtLeast("Should have polled some rows before unsubscribe", 1, polledRows);
+
+ // Unsubscribe (tears down the consensus queue)
+ System.out.println(" Unsubscribing (queue teardown)...");
+ consumer.unsubscribe(topicName);
+ Thread.sleep(2000);
+
+ // Now commit the previously polled messages — should NOT throw
+ System.out.println(
+ " Committing " + uncommittedMessages.size() + " messages AFTER unsubscribe...");
+ boolean commitSucceeded = true;
+ for (SubscriptionMessage msg : uncommittedMessages) {
+ try {
+ consumer.commitSync(msg);
+ } catch (Exception e) {
+ System.out.println(" Commit threw exception: " + e.getMessage());
+ commitSucceeded = false;
+ }
+ }
+
+ System.out.println(" Commit after unsubscribe completed. Success=" + commitSucceeded);
+ assertTrue("Commit after unsubscribe should succeed without exception", commitSucceeded);
+ System.out.println(" (Key: no exception crash, routing handled gracefully)");
+ } finally {
+ if (consumer != null) {
+ try {
+ consumer.close();
+ } catch (Exception e) {
+ /* ignore */
+ }
+ }
+ dropTopicTable(topicName);
+ deleteDatabase(database);
+ }
+ }
+
+ // ======================================================================
+ // Test 8: Seek (seekToBeginning, seekToEnd, seek by timestamp)
+ // ======================================================================
+ /**
+ * Verifies all three seek operations in a single flow:
+ *
+ *
+ * - seekToBeginning — re-delivers previously committed data from earliest available position
+ *
- seekToEnd — skips all existing data, only new writes are received
+ *
- seek(timestamp) — positions at the approximate WAL entry matching the given timestamp
+ *
+ */
+ private static void testSeek() throws Exception {
+ String database = nextDatabase();
+ String topicName = nextTopic();
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId = nextConsumerId();
+ SubscriptionTablePullConsumer consumer = null;
+
+ try {
+ // Step 0: Create DataRegion
+ try (ITableSession session = openTableSession()) {
+ createDatabaseAndTable(session, database, "t1", "tag1 STRING TAG, s1 INT64 FIELD");
+ }
+ Thread.sleep(2000);
+
+ // Step 1: Create topic + consumer + subscribe
+ System.out.println(" Step 1: Create topic and subscribe");
+ createTopicTable(topicName, database, "t1");
+ Thread.sleep(1000);
+
+ consumer = (SubscriptionTablePullConsumer) createConsumer(consumerId, consumerGroupId);
+ consumer.subscribe(topicName);
+ Thread.sleep(3000);
+
+ // Step 2: Write 1000 rows with timestamps 1000..1999 and poll+commit all
+ System.out.println(" Step 2: Write 1000 rows (timestamps 1000..1999) and poll+commit");
+ try (ITableSession session = openTableSession()) {
+ session.executeNonQueryStatement("USE " + database);
+ for (int i = 0; i < 1000; i++) {
+ long ts = 1000 + i;
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO t1 (tag1, s1, time) VALUES ('d1', %d, %d)", ts * 10, ts));
+ }
+ }
+ Thread.sleep(2000);
+
+ PollResult firstPoll = pollUntilComplete(consumer, 1000, 120);
+ System.out.println(" First poll: " + firstPoll.totalRows + " rows");
+ assertAtLeast("First poll should get rows", 1, firstPoll.totalRows);
+
+ // ------------------------------------------------------------------
+ // Step 3: seekToBeginning — should re-deliver data from the start
+ // ------------------------------------------------------------------
+ System.out.println(" Step 3: seekToBeginning → expect re-delivery");
+ consumer.seekToBeginning(topicName);
+ Thread.sleep(2000);
+
+ // No initial INSERT in table test (Step 0 only creates DB+table), so expectedRows=1000
+ PollResult beginningPoll = pollUntilComplete(consumer, 1000, 120);
+ System.out.println(" After seekToBeginning: " + beginningPoll);
+ assertAtLeast(
+ "seekToBeginning should re-deliver rows (WAL retention permitting)",
+ 1,
+ beginningPoll.totalRows);
+
+ // ------------------------------------------------------------------
+ // Step 4: seekToEnd — should receive nothing until new writes
+ // ------------------------------------------------------------------
+ System.out.println(" Step 4: seekToEnd → expect no old data");
+ consumer.seekToEnd(topicName);
+ Thread.sleep(2000);
+
+ PollResult endPoll = new PollResult();
+ int consecutiveEmpty = 0;
+ for (int attempt = 0; attempt < 15; attempt++) {
+ List msgs = consumer.poll(Duration.ofMillis(1000));
+ if (msgs.isEmpty()) {
+ consecutiveEmpty++;
+ if (consecutiveEmpty >= 5) break;
+ Thread.sleep(500);
+ continue;
+ }
+ consecutiveEmpty = 0;
+ for (SubscriptionMessage msg : msgs) {
+ for (SubscriptionResultSet ds : getResultSets(msg)) {
+ while (ds.hasNext()) {
+ ds.next();
+ endPoll.totalRows++;
+ }
+ }
+ consumer.commitSync(msg);
+ }
+ }
+ System.out.println(" After seekToEnd (no new writes): " + endPoll.totalRows + " rows");
+ // May occasionally be 1 due to prefetch thread race; tolerate small values
+ assertTrue("seekToEnd should yield at most 1 row (race tolerance)", endPoll.totalRows <= 1);
+
+ // Write 200 new rows — they should be received
+ System.out.println(" Writing 200 new rows after seekToEnd");
+ try (ITableSession session = openTableSession()) {
+ session.executeNonQueryStatement("USE " + database);
+ for (int i = 2000; i < 2200; i++) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO t1 (tag1, s1, time) VALUES ('d1', %d, %d)", i * 10, i));
+ }
+ }
+ Thread.sleep(2000);
+
+ PollResult afterEndPoll = pollUntilComplete(consumer, 200, 120);
+ System.out.println(" After seekToEnd + new writes: " + afterEndPoll);
+ assertEquals(
+ "Should receive exactly 200 new rows after seekToEnd", 200, afterEndPoll.totalRows);
+
+ // ------------------------------------------------------------------
+ // Step 5: seek(timestamp) — seek to timestamp 1500
+ // ------------------------------------------------------------------
+ System.out.println(" Step 5: seek(1500) → expect rows from near ts=1500");
+ consumer.seekToBeginning(topicName);
+ Thread.sleep(2000);
+
+ PollResult midpointPoll = new PollResult();
+ TopicProgress midpointProgress = null;
+ consecutiveEmpty = 0;
+ for (int attempt = 0; attempt < 20 && midpointProgress == null; attempt++) {
+ List msgs = consumer.poll(Duration.ofMillis(1000));
+ if (msgs.isEmpty()) {
+ consecutiveEmpty++;
+ if (consecutiveEmpty >= 5) break;
+ Thread.sleep(500);
+ continue;
+ }
+ consecutiveEmpty = 0;
+ for (SubscriptionMessage msg : msgs) {
+ for (SubscriptionResultSet ds : getResultSets(msg)) {
+ while (ds.hasNext()) {
+ ds.next();
+ midpointPoll.totalRows++;
+ }
+ }
+ consumer.commitSync(msg);
+ if (midpointPoll.totalRows >= 500) {
+ midpointProgress = consumer.committedPositions(topicName);
+ break;
+ }
+ }
+ }
+ assertTrue("Should capture a midpoint TopicProgress", midpointProgress != null);
+
+ consumer.seek(topicName, midpointProgress);
+ Thread.sleep(2000);
+
+ // Sparse mapping (interval=100) positions near ts=1500.
+ // Expect: ~500 rows from ts≥1500 in original data (1500..1999)
+ // + 200 rows from new writes (2000..2199) = ~700 minimum
+ PollResult afterSeek = pollUntilComplete(consumer, 1200, 120);
+ final int minimumTailRows = Math.max(1, 1200 - midpointPoll.totalRows);
+ System.out.println(
+ " After seek(topicProgress): "
+ + afterSeek.totalRows
+ + " rows from midpoint progress "
+ + midpointPoll.totalRows);
+ assertAtLeast(
+ "seek(topicProgress) should deliver the remaining tail rows",
+ minimumTailRows,
+ afterSeek.totalRows);
+
+ // ------------------------------------------------------------------
+ // Step 6: seek(future timestamp) — expect 0 rows
+ // ------------------------------------------------------------------
+ System.out.println(" Step 6: seek(99999) → expect no data");
+ TopicProgress tailProgress = consumer.committedPositions(topicName);
+ assertTrue("Tail TopicProgress should be available after replay", tailProgress != null);
+ consumer.seekAfter(topicName, tailProgress);
+ Thread.sleep(2000);
+
+ PollResult futurePoll = new PollResult();
+ consecutiveEmpty = 0;
+ for (int attempt = 0; attempt < 10; attempt++) {
+ List msgs = consumer.poll(Duration.ofMillis(1000));
+ if (msgs.isEmpty()) {
+ consecutiveEmpty++;
+ if (consecutiveEmpty >= 5) break;
+ Thread.sleep(500);
+ continue;
+ }
+ consecutiveEmpty = 0;
+ for (SubscriptionMessage msg : msgs) {
+ for (SubscriptionResultSet ds : getResultSets(msg)) {
+ while (ds.hasNext()) {
+ ds.next();
+ futurePoll.totalRows++;
+ }
+ }
+ consumer.commitSync(msg);
+ }
+ }
+ System.out.println(
+ " After seekAfter(tail topicProgress): " + futurePoll.totalRows + " rows");
+ // seek(99999) should behave like seekToEnd — 0 rows normally,
+ // but may yield up to 1 row due to prefetch thread race (same as seekToEnd)
+ assertTrue(
+ "seekAfter(tail topicProgress) should yield at most 1 row (race tolerance)",
+ futurePoll.totalRows <= 1);
+
+ // ------------------------------------------------------------------
+ // Step 7: seek(topicProgress) — seek by per-region writer progress
+ // ------------------------------------------------------------------
+ System.out.println(
+ " Step 7: seekToBeginning first, then poll to collect per-region positions");
+ consumer.seekToBeginning(topicName);
+ Thread.sleep(2000);
+
+ List positionSnapshots = new ArrayList<>();
+ List rowsPerMsg = new ArrayList<>();
+ int totalRowsCollected = 0;
+ consecutiveEmpty = 0;
+
+ for (int attempt = 0; attempt < 60; attempt++) {
+ List msgs = consumer.poll(Duration.ofMillis(2000));
+ if (msgs.isEmpty()) {
+ consecutiveEmpty++;
+ if (consecutiveEmpty >= 5 && totalRowsCollected > 0) break;
+ Thread.sleep(500);
+ continue;
+ }
+ consecutiveEmpty = 0;
+ for (SubscriptionMessage msg : msgs) {
+ int msgRows = 0;
+ for (SubscriptionResultSet ds : getResultSets(msg)) {
+ while (ds.hasNext()) {
+ ds.next();
+ msgRows++;
+ }
+ }
+ consumer.commitSync(msg);
+ rowsPerMsg.add(msgRows);
+ totalRowsCollected += msgRows;
+ positionSnapshots.add(consumer.committedPositions(topicName));
+ }
+ }
+ System.out.println(
+ " Collected "
+ + totalRowsCollected
+ + " rows in "
+ + positionSnapshots.size()
+ + " messages");
+
+ if (positionSnapshots.size() >= 2) {
+ int midIdx = positionSnapshots.size() / 2;
+ TopicProgress seekPositions = positionSnapshots.get(midIdx);
+ int writerFrontierCount = countWriterFrontiers(seekPositions);
+ assertTrue(
+ "committed TopicProgress should contain at least one writer frontier",
+ writerFrontierCount > 0);
+ System.out.println(
+ " seekAfter(topicProgress.regionCount="
+ + seekPositions.getRegionProgress().size()
+ + ", writerFrontierCount="
+ + writerFrontierCount
+ + ") [msg "
+ + midIdx
+ + "/"
+ + positionSnapshots.size()
+ + "]");
+
+ int expectedFromMid = 0;
+ for (int i = midIdx; i < rowsPerMsg.size(); i++) {
+ expectedFromMid += rowsPerMsg.get(i);
+ }
+
+ consumer.seekAfter(topicName, seekPositions);
+ Thread.sleep(2000);
+
+ PollResult afterSeekEpoch = pollUntilComplete(consumer, expectedFromMid, 60);
+ System.out.println(
+ " After seekAfter(topicProgress): "
+ + afterSeekEpoch.totalRows
+ + " rows (expected ~"
+ + expectedFromMid
+ + ")");
+ assertAtLeast(
+ "seekAfter(topicProgress) should deliver at least half the tail data",
+ expectedFromMid / 2,
+ afterSeekEpoch.totalRows);
+ } else {
+ System.out.println(
+ " SKIP seekAfter(topicProgress) sub-test: only "
+ + positionSnapshots.size()
+ + " messages");
+ }
+
+ System.out.println(" testSeek passed all sub-tests!");
+ } finally {
+ cleanup(consumer, topicName, database);
+ }
+ }
+
+ // ======================================================================
+ // Test 9: Processor Framework (ColumnAlignProcessor + WatermarkProcessor + PollResult)
+ // ======================================================================
+ /**
+ * Verifies:
+ *
+ *
+ * - ColumnAlignProcessor forward-fills null columns per table
+ *
- pollWithInfo() returns PollResult with correct metadata
+ *
- WatermarkProcessor buffers and emits based on watermark
+ *
- Processor chaining works correctly
+ *
- Idempotent double-commit does not throw
+ *
+ */
+ private static void testProcessorFramework() throws Exception {
+ String database = nextDatabase();
+ String topicName = nextTopic();
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId = nextConsumerId();
+ String tableName = "proc_test";
+ SubscriptionTablePullConsumer consumer = null;
+ SubscriptionTablePullConsumer consumer2 = null;
+
+ try {
+ // Step 1: Create table with 3 measurement columns
+ System.out.println(" Step 1: Creating table with 3 measurement columns");
+ try (ITableSession session = openTableSession()) {
+ createDatabaseAndTable(
+ session,
+ database,
+ tableName,
+ "device_id STRING TAG, s1 INT32 FIELD, s2 INT32 FIELD, s3 INT32 FIELD");
+ }
+
+ // Step 2: Create topic and subscribe
+ System.out.println(" Step 2: Creating topic and subscribing");
+ createTopicTable(topicName, database, tableName);
+ Thread.sleep(1000);
+
+ // Build consumer with ColumnAlignProcessor — use concrete type for addProcessor access
+ consumer =
+ (SubscriptionTablePullConsumer)
+ new SubscriptionTablePullConsumerBuilder()
+ .host(HOST)
+ .port(PORT)
+ .consumerId(consumerId)
+ .consumerGroupId(consumerGroupId)
+ .autoCommit(false)
+ .build();
+ consumer.addProcessor(new ColumnAlignProcessor());
+ consumer.open();
+ consumer.subscribe(topicName);
+ Thread.sleep(3000);
+
+ // Step 3: Write a Tablet with 2 rows — row 2 has s2/s3 null (marked in BitMap).
+ // Using insertTablet ensures both rows share the same Tablet with all 3 columns,
+ // so ColumnAlignProcessor can forward-fill the nulls.
+ System.out.println(" Step 3: Writing partial-column data via insertTablet");
+ try (ITableSession session = openTableSession()) {
+ session.executeNonQueryStatement("USE " + database);
+ List schemas =
+ Arrays.asList(
+ new MeasurementSchema("device_id", TSDataType.STRING),
+ new MeasurementSchema("s1", TSDataType.INT32),
+ new MeasurementSchema("s2", TSDataType.INT32),
+ new MeasurementSchema("s3", TSDataType.INT32));
+ List categories =
+ Arrays.asList(
+ ColumnCategory.TAG,
+ ColumnCategory.FIELD,
+ ColumnCategory.FIELD,
+ ColumnCategory.FIELD);
+ Tablet tablet =
+ new Tablet(
+ tableName,
+ IMeasurementSchema.getMeasurementNameList(schemas),
+ IMeasurementSchema.getDataTypeList(schemas),
+ categories,
+ 2);
+
+ // Row 0 (time=100): all columns present
+ tablet.addTimestamp(0, 100);
+ tablet.addValue("device_id", 0, "dev1");
+ tablet.addValue("s1", 0, 10);
+ tablet.addValue("s2", 0, 20);
+ tablet.addValue("s3", 0, 30);
+
+ // Row 1 (time=200): only s1 — s2/s3 remain null (BitMap marked by addTimestamp)
+ tablet.addTimestamp(1, 200);
+ tablet.addValue("device_id", 1, "dev1");
+ tablet.addValue("s1", 1, 11);
+
+ session.insert(tablet);
+ session.executeNonQueryStatement("FLUSH");
+ }
+ Thread.sleep(2000);
+
+ // Step 4: Poll with pollWithInfo and verify ColumnAlign + PollResult
+ System.out.println(" Step 4: Polling with pollWithInfo");
+ int totalRows = 0;
+ boolean foundForwardFill = false;
+ org.apache.iotdb.session.subscription.payload.PollResult lastPollResult = null;
+ List allMessages = new ArrayList<>();
+
+ for (int attempt = 0; attempt < 30; attempt++) {
+ org.apache.iotdb.session.subscription.payload.PollResult pollResult =
+ consumer.pollWithInfo(Duration.ofMillis(1000));
+ lastPollResult = pollResult;
+
+ assertTrue("PollResult should not be null", pollResult != null);
+ // With only ColumnAlignProcessor (non-buffering), bufferedCount should be 0
+ assertEquals("ColumnAlignProcessor should not buffer", 0, pollResult.getBufferedCount());
+
+ List msgs = pollResult.getMessages();
+ if (msgs.isEmpty()) {
+ if (totalRows >= 2) break;
+ Thread.sleep(1000);
+ continue;
+ }
+
+ allMessages.addAll(msgs);
+ for (SubscriptionMessage msg : msgs) {
+ for (SubscriptionResultSet ds : getResultSets(msg)) {
+ List columnNames = ds.getColumnNames();
+ while (ds.hasNext()) {
+ org.apache.tsfile.read.common.RowRecord row = ds.nextRecord();
+ totalRows++;
+ List fields = row.getFields();
+ System.out.println(
+ " Row: time="
+ + row.getTimestamp()
+ + ", columns="
+ + columnNames
+ + ", fields="
+ + fields);
+ // Check forward-fill: at timestamp 200, s2 and s3 should be filled
+ if (row.getTimestamp() == 200) {
+ // Table results include "time" in columnNames but not in fields.
+ int s2ColumnIdx = columnNames.indexOf("s2");
+ int s3ColumnIdx = columnNames.indexOf("s3");
+ int fieldOffset =
+ !columnNames.isEmpty() && "time".equalsIgnoreCase(columnNames.get(0)) ? 1 : 0;
+ int s2FieldIdx = s2ColumnIdx - fieldOffset;
+ int s3FieldIdx = s3ColumnIdx - fieldOffset;
+ if (s2FieldIdx >= 0
+ && s3FieldIdx >= 0
+ && s2FieldIdx < fields.size()
+ && s3FieldIdx < fields.size()
+ && fields.get(s2FieldIdx) != null
+ && fields.get(s2FieldIdx).getDataType() != null
+ && fields.get(s3FieldIdx) != null
+ && fields.get(s3FieldIdx).getDataType() != null) {
+ foundForwardFill = true;
+ System.out.println(" >>> Forward-fill confirmed at timestamp 200");
+ }
+ }
+ }
+ }
+ }
+ }
+
+ assertEquals("Expected 2 rows total", 2, totalRows);
+ assertTrue(
+ "ColumnAlignProcessor should forward-fill nulls at timestamp 200", foundForwardFill);
+ System.out.println(" ColumnAlignProcessor: PASSED");
+
+ // Step 5: Idempotent double-commit
+ System.out.println(" Step 5: Testing idempotent double-commit");
+ if (!allMessages.isEmpty()) {
+ SubscriptionMessage firstMsg = allMessages.get(0);
+ consumer.commitSync(firstMsg);
+ // Second commit of same message should not throw
+ consumer.commitSync(firstMsg);
+ System.out.println(" Double-commit succeeded (idempotent)");
+ }
+
+ // Step 6: Test with WatermarkProcessor chained
+ System.out.println(" Step 6: Verifying WatermarkProcessor buffering");
+ // Close current consumer and create a new one with WatermarkProcessor
+ consumer.unsubscribe(topicName);
+ consumer.close();
+
+ String consumerId2 = consumerId + "_wm";
+ consumer2 =
+ (SubscriptionTablePullConsumer)
+ new SubscriptionTablePullConsumerBuilder()
+ .host(HOST)
+ .port(PORT)
+ .consumerId(consumerId2)
+ .consumerGroupId(consumerGroupId + "_wm")
+ .autoCommit(false)
+ .build();
+ // Chain: ColumnAlign → Watermark(5s out-of-order, 10s timeout)
+ consumer2.addProcessor(new ColumnAlignProcessor());
+ consumer2.addProcessor(new WatermarkProcessor(5000, 10000));
+ consumer2.open();
+ consumer2.subscribe(topicName);
+ Thread.sleep(3000);
+
+ // Write data that should be buffered by watermark
+ try (ITableSession session = openTableSession()) {
+ session.executeNonQueryStatement("USE " + database);
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO %s(time, device_id, s1, s2, s3) VALUES (1000, 'dev1', 100, 200, 300)",
+ tableName));
+ session.executeNonQueryStatement("FLUSH");
+ }
+ Thread.sleep(2000);
+
+ // First poll — data may be buffered by WatermarkProcessor
+ org.apache.iotdb.session.subscription.payload.PollResult wmResult =
+ consumer2.pollWithInfo(Duration.ofMillis(2000));
+ System.out.println(
+ " WatermarkProcessor poll: messages="
+ + wmResult.getMessages().size()
+ + ", buffered="
+ + wmResult.getBufferedCount());
+ // The watermark processor may buffer or emit depending on timing;
+ // we just verify the API works and returns valid metadata
+ assertTrue("PollResult bufferedCount should be >= 0", wmResult.getBufferedCount() >= 0);
+
+ // consumer already closed above in Step 6 setup
+ consumer = null;
+
+ System.out.println(" testProcessorFramework passed all sub-tests!");
+ } finally {
+ cleanup(consumer, topicName, database);
+ cleanup(consumer2, topicName, database);
+ }
+ }
+
+ // ======================================================================
+ // Test 10: Poison Message Drop — messages nacked beyond threshold
+ // are force-acked (dropped) and don't block new data.
+ // ======================================================================
+ /**
+ * Verifies:
+ *
+ *
+ * - A message nacked more than POISON_MESSAGE_NACK_THRESHOLD (10) times is dropped
+ *
- After drop, new data can still be received
+ *
- The consumer is not permanently blocked by a single unprocessable message
+ *
+ */
+ private static void testPoisonMessageDrop() throws Exception {
+ String database = nextDatabase();
+ String topicName = nextTopic();
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId = nextConsumerId();
+ SubscriptionTablePullConsumer consumer = null;
+
+ try {
+ // Step 0: Create DataRegion
+ try (ITableSession session = openTableSession()) {
+ createDatabaseAndTable(session, database, "t1", "tag1 STRING TAG, s1 INT64 FIELD");
+ }
+ Thread.sleep(2000);
+
+ // Step 1: Create topic and subscribe
+ System.out.println(" Step 1: Creating topic and subscribing");
+ createTopicTable(topicName, database, "t1");
+ Thread.sleep(1000);
+
+ consumer = (SubscriptionTablePullConsumer) createConsumer(consumerId, consumerGroupId);
+ consumer.subscribe(topicName);
+ Thread.sleep(3000);
+
+ // Step 2: Write initial data that will become the "poison" message
+ System.out.println(" Step 2: Writing 10 rows (the initial batch)");
+ try (ITableSession session = openTableSession()) {
+ session.executeNonQueryStatement("USE " + database);
+ for (int i = 1; i <= 10; i++) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO t1 (tag1, s1, time) VALUES ('d1', %d, %d)", i * 10, i));
+ }
+ }
+ Thread.sleep(2000);
+
+ // Step 3: Poll without commit — repeatedly. Each poll-then-timeout cycle
+ // causes the server to nack the in-flight event and re-enqueue it.
+ System.out.println(
+ " Step 3: Polling without commit for 15 rounds (threshold=10, need >10 nacks)");
+ int totalPoisonPolled = 0;
+ for (int round = 1; round <= 15; round++) {
+ List msgs = consumer.poll(Duration.ofMillis(3000));
+ int roundRows = 0;
+ for (SubscriptionMessage msg : msgs) {
+ for (SubscriptionResultSet ds : getResultSets(msg)) {
+ while (ds.hasNext()) {
+ ds.next();
+ roundRows++;
+ totalPoisonPolled++;
+ }
+ }
+ // Deliberately NOT committing — this is the "nack" behavior
+ }
+ System.out.println(
+ " Round " + round + ": received " + roundRows + " rows (NOT committing)");
+ if (msgs.isEmpty() && round > 11) {
+ System.out.println(" No messages — poison message may have been force-acked");
+ break;
+ }
+ Thread.sleep(1000);
+ }
+ System.out.println(" Total rows polled across all rounds: " + totalPoisonPolled);
+
+ // Step 4: Write NEW data and verify it can be received (consumer not blocked)
+ System.out.println(" Step 4: Writing 50 NEW rows and polling WITH commit");
+ try (ITableSession session = openTableSession()) {
+ session.executeNonQueryStatement("USE " + database);
+ for (int i = 1000; i < 1050; i++) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO t1 (tag1, s1, time) VALUES ('d1', %d, %d)", i * 10, i));
+ }
+ }
+ Thread.sleep(2000);
+
+ PollResult newResult = pollUntilComplete(consumer, 50, 60);
+ System.out.println(" New data poll result: " + newResult);
+
+ assertAtLeast(
+ "Consumer must not be permanently blocked by poison message — new data should arrive",
+ 1,
+ newResult.totalRows);
+ System.out.println(
+ " testPoisonMessageDrop passed: consumer received "
+ + newResult.totalRows
+ + " new rows after poison message handling");
+ } finally {
+ cleanup(consumer, topicName, database);
+ }
+ }
+
+ // ======================================================================
+ // Test 11: Serialization V2 Fields — regionId, epoch, dataNodeId
+ // are properly populated in polled messages' SubscriptionCommitContext.
+ // ======================================================================
+ /**
+ * Verifies:
+ *
+ *
+ * - SubscriptionCommitContext.getWriterId() is non-null for consensus messages
+ *
- SubscriptionCommitContext.getWriterProgress() is non-null for consensus messages
+ *
- SubscriptionCommitContext.getWriterId().getRegionId() stays aligned with the region
+ *
- These writer-progress fields survive the serialize/deserialize round-trip through RPC
+ *
+ */
+ private static void testWriterProgressFields() throws Exception {
+ String database = nextDatabase();
+ String topicName = nextTopic();
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId = nextConsumerId();
+ SubscriptionTablePullConsumer consumer = null;
+
+ try {
+ // Step 0: Create DataRegion
+ try (ITableSession session = openTableSession()) {
+ createDatabaseAndTable(session, database, "t1", "tag1 STRING TAG, s1 INT64 FIELD");
+ }
+ Thread.sleep(2000);
+
+ // Step 1: Create topic and subscribe
+ System.out.println(" Step 1: Creating topic and subscribing");
+ createTopicTable(topicName, database, "t1");
+ Thread.sleep(1000);
+
+ consumer = (SubscriptionTablePullConsumer) createConsumer(consumerId, consumerGroupId);
+ consumer.subscribe(topicName);
+ Thread.sleep(3000);
+
+ // Step 2: Write data
+ System.out.println(" Step 2: Writing 20 rows");
+ try (ITableSession session = openTableSession()) {
+ session.executeNonQueryStatement("USE " + database);
+ for (int i = 1; i <= 20; i++) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO t1 (tag1, s1, time) VALUES ('d1', %d, %d)", i * 10, i));
+ }
+ }
+ Thread.sleep(2000);
+
+ // Step 3: Poll and check writer-progress fields in SubscriptionCommitContext
+ System.out.println(" Step 3: Polling and verifying writer-progress fields in CommitContext");
+ int totalRows = 0;
+ int messagesChecked = 0;
+ boolean foundWriterProgress = false;
+
+ for (int attempt = 0; attempt < 30; attempt++) {
+ List msgs = consumer.poll(Duration.ofMillis(2000));
+ if (msgs.isEmpty()) {
+ if (totalRows > 0) break;
+ Thread.sleep(1000);
+ continue;
+ }
+
+ for (SubscriptionMessage msg : msgs) {
+ SubscriptionCommitContext ctx = msg.getCommitContext();
+ messagesChecked++;
+
+ // Check writer-progress fields and their compatibility projections
+ String regionId = ctx.getRegionId();
+ int dataNodeId = ctx.getDataNodeId();
+ WriterId writerId = ctx.getWriterId();
+ WriterProgress writerProgress = ctx.getWriterProgress();
+ long physicalTime =
+ writerProgress != null ? writerProgress.getPhysicalTime() : Long.MIN_VALUE;
+
+ System.out.println(
+ " Message "
+ + messagesChecked
+ + ": regionId="
+ + regionId
+ + ", physicalTime="
+ + physicalTime
+ + ", writerId="
+ + writerId
+ + ", writerProgress="
+ + writerProgress
+ + ", dataNodeId="
+ + dataNodeId
+ + ", topicName="
+ + ctx.getTopicName()
+ + ", consumerGroupId="
+ + ctx.getConsumerGroupId());
+
+ assertTrue(
+ "regionId should be non-null for consensus message",
+ regionId != null && !regionId.isEmpty());
+ assertTrue("writerId should be non-null for consensus message", writerId != null);
+ assertTrue(
+ "writerProgress should be non-null for consensus message", writerProgress != null);
+ assertEquals("regionId should match writerId.regionId", writerId.getRegionId(), regionId);
+ assertEquals(
+ "physicalTime should mirror writerProgress.physicalTime",
+ writerProgress.getPhysicalTime(),
+ physicalTime);
+ foundWriterProgress = true;
+
+ assertTrue("physicalTime should be >= 0, got " + physicalTime, physicalTime >= 0);
+
+ assertTrue("dataNodeId should be > 0, got " + dataNodeId, dataNodeId > 0);
+
+ for (SubscriptionResultSet ds : getResultSets(msg)) {
+ while (ds.hasNext()) {
+ ds.next();
+ totalRows++;
+ }
+ }
+ consumer.commitSync(msg);
+ }
+ }
+
+ System.out.println(
+ " Checked "
+ + messagesChecked
+ + " messages, "
+ + totalRows
+ + " rows. foundWriterProgress="
+ + foundWriterProgress);
+ assertAtLeast("Should have received data rows", 1, totalRows);
+ assertTrue(
+ "Should have found writer-progress metadata in at least one message",
+ foundWriterProgress);
+ System.out.println(" testWriterProgressFields passed!");
+ } finally {
+ cleanup(consumer, topicName, database);
+ }
+ }
+
+ private static List getResultSets(final SubscriptionMessage message) {
+ return message.getResultSets().stream()
+ .map(resultSet -> (SubscriptionResultSet) resultSet)
+ .collect(Collectors.toList());
+ }
+}
diff --git a/example/subscription/src/main/java/org/apache/iotdb/ConsensusSubscriptionTest.java b/example/subscription/src/main/java/org/apache/iotdb/ConsensusSubscriptionTest.java
new file mode 100644
index 0000000000000..d2c4044ca908a
--- /dev/null
+++ b/example/subscription/src/main/java/org/apache/iotdb/ConsensusSubscriptionTest.java
@@ -0,0 +1,2018 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb;
+
+import org.apache.iotdb.isession.ISession;
+import org.apache.iotdb.rpc.subscription.config.TopicConstant;
+import org.apache.iotdb.rpc.subscription.payload.poll.RegionProgress;
+import org.apache.iotdb.rpc.subscription.payload.poll.TopicProgress;
+import org.apache.iotdb.session.Session;
+import org.apache.iotdb.session.subscription.SubscriptionTreeSession;
+import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer;
+import org.apache.iotdb.session.subscription.payload.SubscriptionMessage;
+import org.apache.iotdb.session.subscription.payload.SubscriptionRecordHandler.SubscriptionResultSet;
+
+import org.apache.tsfile.common.conf.TSFileConfig;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
+/** TODO: move these manual tests into ITs */
+public class ConsensusSubscriptionTest {
+
+ private static final String HOST = "127.0.0.1";
+ private static final int PORT = 6667;
+ private static final String USER = "root";
+ private static final String PASSWORD = "root";
+
+ private static int testCounter = 0;
+ private static int passed = 0;
+ private static int failed = 0;
+ private static final List failedTests = new ArrayList<>();
+
+ public static void main(String[] args) throws Exception {
+ System.out.println("=== Consensus-Based Subscription Test Suite ===\n");
+
+ String targetTest = args.length > 0 ? args[0] : null;
+
+ if (targetTest == null || "testBasicFlow".equals(targetTest)) {
+ runTest("testBasicFlow", ConsensusSubscriptionTest::testBasicFlow);
+ }
+ if (targetTest == null || "testDataTypes".equals(targetTest)) {
+ runTest("testDataTypes", ConsensusSubscriptionTest::testDataTypes);
+ }
+ if (targetTest == null || "testFilteringAndTopicSelection".equals(targetTest)) {
+ runTest(
+ "testFilteringAndTopicSelection",
+ ConsensusSubscriptionTest::testFilteringAndTopicSelection);
+ }
+ if (targetTest == null || "testSubscribeBeforeRegion".equals(targetTest)) {
+ runTest("testSubscribeBeforeRegion", ConsensusSubscriptionTest::testSubscribeBeforeRegion);
+ }
+ if (targetTest == null || "testMultiEntityIsolation".equals(targetTest)) {
+ runTest("testMultiEntityIsolation", ConsensusSubscriptionTest::testMultiEntityIsolation);
+ }
+ if (targetTest == null || "testWalCatchUpAndGapRecovery".equals(targetTest)) {
+ runTest(
+ "testWalCatchUpAndGapRecovery", ConsensusSubscriptionTest::testWalCatchUpAndGapRecovery);
+ }
+ if (targetTest == null || "testSeekAndPositionSemantics".equals(targetTest)) {
+ runTest(
+ "testSeekAndPositionSemantics", ConsensusSubscriptionTest::testSeekAndPositionSemantics);
+ }
+ if (targetTest == null || "testConsumerRestartRecovery".equals(targetTest)) {
+ runTest(
+ "testConsumerRestartRecovery", ConsensusSubscriptionTest::testConsumerRestartRecovery);
+ }
+ if (targetTest == null || "testAckNackAndPoisonSemantics".equals(targetTest)) {
+ runTest(
+ "testAckNackAndPoisonSemantics",
+ ConsensusSubscriptionTest::testAckNackAndPoisonSemantics);
+ }
+
+ // Summary
+ System.out.println("\n=== Test Suite Summary ===");
+ System.out.println("Passed: " + passed);
+ System.out.println("Failed: " + failed);
+ if (!failedTests.isEmpty()) {
+ System.out.println("Failed tests: " + failedTests);
+ }
+ System.out.println("=== Done ===");
+ }
+
+ // ============================
+ // Test Infrastructure
+ // ============================
+
+ @FunctionalInterface
+ interface TestMethod {
+ void run() throws Exception;
+ }
+
+ private static void runTest(String name, TestMethod test) {
+ System.out.println("\n" + "=================================================================");
+ System.out.println("Running: " + name);
+ System.out.println("=================================================================");
+ try {
+ test.run();
+ passed++;
+ System.out.println(">>> PASSED: " + name);
+ } catch (AssertionError e) {
+ failed++;
+ failedTests.add(name);
+ System.out.println(">>> FAILED: " + name + " - " + e.getMessage());
+ e.printStackTrace(System.out);
+ } catch (Exception e) {
+ failed++;
+ failedTests.add(name);
+ System.out.println(">>> ERROR: " + name + " - " + e.getMessage());
+ e.printStackTrace(System.out);
+ }
+ }
+
+ private static String nextDatabase() {
+ testCounter++;
+ return "root.csub_test_" + testCounter;
+ }
+
+ private static String nextTopic() {
+ return "topic_csub_" + testCounter;
+ }
+
+ private static String nextConsumerGroup() {
+ return "cg_csub_" + testCounter;
+ }
+
+ private static String nextConsumerId() {
+ return "consumer_csub_" + testCounter;
+ }
+
+ private static ISession openSession() throws Exception {
+ ISession session =
+ new Session.Builder().host(HOST).port(PORT).username(USER).password(PASSWORD).build();
+ session.open();
+ return session;
+ }
+
+ private static void createDatabase(ISession session, String database) throws Exception {
+ try {
+ session.executeNonQueryStatement("CREATE DATABASE " + database);
+ } catch (Exception e) {
+ // ignore if already exists
+ }
+ }
+
+ private static void deleteDatabase(String database) {
+ try (ISession session = openSession()) {
+ session.executeNonQueryStatement("DELETE DATABASE " + database);
+ } catch (Exception e) {
+ // ignore
+ }
+ }
+
+ private static void dropTopic(String topicName) {
+ try (SubscriptionTreeSession subSession = new SubscriptionTreeSession(HOST, PORT)) {
+ subSession.open();
+ subSession.dropTopic(topicName);
+ } catch (Exception e) {
+ // ignore
+ }
+ }
+
+ private static void createTopic(String topicName, String path) throws Exception {
+ try (SubscriptionTreeSession subSession = new SubscriptionTreeSession(HOST, PORT)) {
+ subSession.open();
+ try {
+ subSession.dropTopic(topicName);
+ } catch (Exception e) {
+ // ignore
+ }
+
+ Properties topicConfig = new Properties();
+ topicConfig.put(TopicConstant.MODE_KEY, TopicConstant.MODE_LIVE_VALUE);
+ topicConfig.put(TopicConstant.FORMAT_KEY, TopicConstant.FORMAT_RECORD_HANDLER_VALUE);
+ topicConfig.put(TopicConstant.PATH_KEY, path);
+ topicConfig.put(TopicConstant.ORDER_MODE_KEY, TopicConstant.ORDER_MODE_PER_WRITER_VALUE);
+ subSession.createTopic(topicName, topicConfig);
+ System.out.println(" Created topic: " + topicName + " (path=" + path + ")");
+ }
+ }
+
+ private static SubscriptionTreePullConsumer createConsumer(
+ String consumerId, String consumerGroupId) throws Exception {
+ SubscriptionTreePullConsumer consumer =
+ new SubscriptionTreePullConsumer.Builder()
+ .host(HOST)
+ .port(PORT)
+ .consumerId(consumerId)
+ .consumerGroupId(consumerGroupId)
+ .autoCommit(false)
+ .buildPullConsumer();
+ consumer.open();
+ return consumer;
+ }
+
+ // ============================
+ // Polling & Verification
+ // ============================
+
+ /**
+ * Poll and commit messages. After reaching expectedRows, continues polling for 5 consecutive
+ * empty rounds to verify no extra data arrives.
+ */
+ private static PollResult pollUntilComplete(
+ SubscriptionTreePullConsumer consumer, int expectedRows, int maxPollAttempts) {
+ return pollUntilComplete(consumer, expectedRows, maxPollAttempts, 1000, true);
+ }
+
+ private static PollResult pollUntilComplete(
+ SubscriptionTreePullConsumer consumer,
+ int expectedRows,
+ int maxPollAttempts,
+ long pollTimeoutMs,
+ boolean commitMessages) {
+ PollResult result = new PollResult();
+ int consecutiveEmpty = 0;
+
+ for (int attempt = 1; attempt <= maxPollAttempts; attempt++) {
+ List messages = consumer.poll(Duration.ofMillis(pollTimeoutMs));
+
+ if (messages.isEmpty()) {
+ consecutiveEmpty++;
+ // Normal completion: reached expected rows and verified quiescence
+ if (consecutiveEmpty >= 3 && result.totalRows >= expectedRows) {
+ System.out.println(
+ " Verified: "
+ + consecutiveEmpty
+ + " consecutive empty polls after "
+ + result.totalRows
+ + " rows (expected "
+ + expectedRows
+ + ")");
+ break;
+ }
+ // Stuck: have data but cannot reach expected count
+ if (consecutiveEmpty >= 5 && result.totalRows > 0) {
+ System.out.println(
+ " Stuck: "
+ + consecutiveEmpty
+ + " consecutive empty polls at "
+ + result.totalRows
+ + " rows (expected "
+ + expectedRows
+ + ")");
+ break;
+ }
+ // Never received anything
+ if (consecutiveEmpty >= 10 && result.totalRows == 0 && expectedRows > 0) {
+ System.out.println(" No data received after " + consecutiveEmpty + " polls");
+ break;
+ }
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException ignored) {
+ }
+ continue;
+ }
+
+ consecutiveEmpty = 0;
+
+ for (SubscriptionMessage message : messages) {
+ for (SubscriptionResultSet dataSet : getResultSets(message)) {
+ String device = null;
+ List columnNames = dataSet.getColumnNames();
+ if (columnNames.size() > 1) {
+ String fullPath = columnNames.get(1);
+ int lastDot = fullPath.lastIndexOf('.');
+ device = lastDot > 0 ? fullPath.substring(0, lastDot) : fullPath;
+ }
+
+ while (dataSet.hasNext()) {
+ try {
+ org.apache.tsfile.read.common.RowRecord record = dataSet.nextRecord();
+ result.totalRows++;
+ if (device != null) {
+ result.rowsPerDevice.merge(device, 1, Integer::sum);
+ }
+ for (int i = 1; i < columnNames.size(); i++) {
+ result.seenColumns.add(columnNames.get(i));
+ }
+ if (result.totalRows <= 5) {
+ System.out.println(
+ " Row: time="
+ + record.getTimestamp()
+ + ", values="
+ + record.getFields()
+ + ", device="
+ + device);
+ }
+ } catch (java.io.IOException e) {
+ throw new RuntimeException("Failed to iterate subscription result set", e);
+ }
+ }
+ }
+ if (commitMessages) {
+ consumer.commitSync(message);
+ }
+ }
+
+ System.out.println(
+ " Poll attempt "
+ + attempt
+ + ": totalRows="
+ + result.totalRows
+ + " / expected="
+ + expectedRows);
+
+ // Stop immediately if we exceeded the expected row count
+ if (expectedRows > 0 && result.totalRows > expectedRows) {
+ System.out.println(
+ " EXCEEDED: totalRows=" + result.totalRows + " > expectedRows=" + expectedRows);
+ break;
+ }
+ }
+
+ return result;
+ }
+
+ // ============================
+ // Cleanup
+ // ============================
+
+ /** Clean up all test artifacts: unsubscribe, close consumer, drop topic, delete database. */
+ private static void cleanup(
+ SubscriptionTreePullConsumer consumer, String topicName, String database) {
+ if (consumer != null) {
+ try {
+ consumer.unsubscribe(topicName);
+ } catch (Exception e) {
+ // ignore
+ }
+ try {
+ consumer.close();
+ } catch (Exception e) {
+ // ignore
+ }
+ }
+ dropTopic(topicName);
+ deleteDatabase(database);
+ }
+
+ // ============================
+ // Result & Assertions
+ // ============================
+
+ static class PollResult {
+ int totalRows = 0;
+ Map rowsPerDevice = new HashMap<>();
+ Set seenColumns = new HashSet<>();
+
+ @Override
+ public String toString() {
+ return "PollResult{totalRows="
+ + totalRows
+ + ", rowsPerDevice="
+ + rowsPerDevice
+ + ", seenColumns="
+ + seenColumns
+ + "}";
+ }
+ }
+
+ private static void assertEquals(String msg, int expected, int actual) {
+ if (expected != actual) {
+ throw new AssertionError(msg + ": expected=" + expected + ", actual=" + actual);
+ }
+ }
+
+ private static void assertEquals(String msg, long expected, long actual) {
+ if (expected != actual) {
+ throw new AssertionError(msg + ": expected=" + expected + ", actual=" + actual);
+ }
+ }
+
+ private static void assertEquals(String msg, String expected, String actual) {
+ if (expected == null ? actual != null : !expected.equals(actual)) {
+ throw new AssertionError(msg + ": expected=" + expected + ", actual=" + actual);
+ }
+ }
+
+ private static void assertTrue(String msg, boolean condition) {
+ if (!condition) {
+ throw new AssertionError(msg);
+ }
+ }
+
+ private static void assertAtLeast(String msg, int min, int actual) {
+ if (actual < min) {
+ throw new AssertionError(msg + ": expected at least " + min + ", actual=" + actual);
+ }
+ }
+
+ private static void assertAtMost(String msg, int max, int actual) {
+ if (actual > max) {
+ throw new AssertionError(msg + ": expected at most " + max + ", actual=" + actual);
+ }
+ }
+
+ private static int countWriterFrontiers(TopicProgress topicProgress) {
+ int writerCount = 0;
+ if (topicProgress == null || topicProgress.getRegionProgress() == null) {
+ return 0;
+ }
+ for (Map.Entry entry : topicProgress.getRegionProgress().entrySet()) {
+ if (entry.getValue() != null && entry.getValue().getWriterPositions() != null) {
+ writerCount += entry.getValue().getWriterPositions().size();
+ }
+ }
+ return writerCount;
+ }
+
+ private static int countRows(SubscriptionMessage message) {
+ int rows = 0;
+ for (SubscriptionResultSet dataSet : getResultSets(message)) {
+ while (dataSet.hasNext()) {
+ try {
+ dataSet.next();
+ rows++;
+ } catch (java.io.IOException e) {
+ throw new RuntimeException("Failed to count rows from subscription result set", e);
+ }
+ }
+ }
+ return rows;
+ }
+
+ private static final class CommittedSnapshot {
+ private final TopicProgress progress;
+ private final int rowsInMessage;
+ private final int cumulativeRows;
+
+ private CommittedSnapshot(TopicProgress progress, int rowsInMessage, int cumulativeRows) {
+ this.progress = progress;
+ this.rowsInMessage = rowsInMessage;
+ this.cumulativeRows = cumulativeRows;
+ }
+ }
+
+ private static final class PolledMessageBatch {
+ private final List messages;
+ private final int totalRows;
+
+ private PolledMessageBatch(List messages, int totalRows) {
+ this.messages = messages;
+ this.totalRows = totalRows;
+ }
+ }
+
+ private static void pause(long millis) {
+ try {
+ Thread.sleep(millis);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException("Interrupted while waiting for subscription test state", e);
+ }
+ }
+
+ private static void bootstrapSeekTopic(String database, String topicName) throws Exception {
+ try (ISession session = openSession()) {
+ createDatabase(session, database);
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d1(time, s1) VALUES (0, 0)", database));
+ session.executeNonQueryStatement("flush");
+ }
+ pause(2000);
+
+ createTopic(topicName, database + ".**");
+ pause(1000);
+ }
+
+ private static SubscriptionTreePullConsumer createSubscribedConsumer(
+ String topicName, String consumerId, String consumerGroupId) throws Exception {
+ SubscriptionTreePullConsumer consumer = createConsumer(consumerId, consumerGroupId);
+ consumer.subscribe(topicName);
+ pause(3000);
+ return consumer;
+ }
+
+ private static void writeSequentialRowsAndFlush(
+ String database, int startTimestampInclusive, int rowCount) throws Exception {
+ try (ISession session = openSession()) {
+ for (int i = 0; i < rowCount; i++) {
+ long ts = startTimestampInclusive + i;
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d1(time, s1) VALUES (%d, %d)", database, ts, ts * 10));
+ }
+ session.executeNonQueryStatement("flush");
+ }
+ pause(2000);
+ }
+
+ private static CommittedSnapshot pollUntilCommittedRows(
+ SubscriptionTreePullConsumer consumer,
+ String topicName,
+ int minimumRows,
+ int maxPollAttempts,
+ long pollTimeoutMs) {
+ int cumulativeRows = 0;
+ int consecutiveEmpty = 0;
+
+ for (int attempt = 1; attempt <= maxPollAttempts; attempt++) {
+ List messages = consumer.poll(Duration.ofMillis(pollTimeoutMs));
+ if (messages.isEmpty()) {
+ consecutiveEmpty++;
+ if (consecutiveEmpty >= 5 && cumulativeRows > 0) {
+ break;
+ }
+ pause(1000);
+ continue;
+ }
+
+ consecutiveEmpty = 0;
+ for (SubscriptionMessage message : messages) {
+ int rowsInMessage = countRows(message);
+ consumer.commitSync(message);
+ cumulativeRows += rowsInMessage;
+ TopicProgress checkpoint = consumer.committedPositions(topicName);
+ System.out.println(
+ " Captured committed checkpoint after "
+ + cumulativeRows
+ + " rows (last message="
+ + rowsInMessage
+ + ")");
+ if (cumulativeRows >= minimumRows) {
+ return new CommittedSnapshot(checkpoint, rowsInMessage, cumulativeRows);
+ }
+ }
+ }
+
+ throw new AssertionError(
+ "Unable to capture committed checkpoint after "
+ + minimumRows
+ + " rows; stopped at "
+ + cumulativeRows);
+ }
+
+ private static List pollAndCaptureCommittedSnapshots(
+ SubscriptionTreePullConsumer consumer,
+ String topicName,
+ int maxPollAttempts,
+ long pollTimeoutMs) {
+ List snapshots = new ArrayList<>();
+ int cumulativeRows = 0;
+ int consecutiveEmpty = 0;
+
+ for (int attempt = 1; attempt <= maxPollAttempts; attempt++) {
+ List messages = consumer.poll(Duration.ofMillis(pollTimeoutMs));
+ if (messages.isEmpty()) {
+ consecutiveEmpty++;
+ if (consecutiveEmpty >= 3 && cumulativeRows > 0) {
+ break;
+ }
+ if (consecutiveEmpty >= 8 && cumulativeRows == 0) {
+ break;
+ }
+ pause(1000);
+ continue;
+ }
+
+ consecutiveEmpty = 0;
+ for (SubscriptionMessage message : messages) {
+ int rowsInMessage = countRows(message);
+ consumer.commitSync(message);
+ cumulativeRows += rowsInMessage;
+ snapshots.add(
+ new CommittedSnapshot(
+ consumer.committedPositions(topicName), rowsInMessage, cumulativeRows));
+ }
+ }
+
+ System.out.println(
+ " Drained "
+ + cumulativeRows
+ + " rows across "
+ + snapshots.size()
+ + " committed messages");
+ return snapshots;
+ }
+
+ private static PolledMessageBatch pollFirstNonEmptyBatchWithoutCommit(
+ SubscriptionTreePullConsumer consumer, int maxPollAttempts, long pollTimeoutMs) {
+ for (int attempt = 1; attempt <= maxPollAttempts; attempt++) {
+ List messages = consumer.poll(Duration.ofMillis(pollTimeoutMs));
+ if (messages.isEmpty()) {
+ pause(1000);
+ continue;
+ }
+
+ int totalRows = 0;
+ for (SubscriptionMessage message : messages) {
+ totalRows += countRows(message);
+ }
+ System.out.println(
+ " Polled stale batch without commit: "
+ + messages.size()
+ + " messages, "
+ + totalRows
+ + " rows");
+ return new PolledMessageBatch(new ArrayList<>(messages), totalRows);
+ }
+
+ return new PolledMessageBatch(new ArrayList<>(), 0);
+ }
+
+ private static int totalRows(List snapshots) {
+ return snapshots.isEmpty() ? 0 : snapshots.get(snapshots.size() - 1).cumulativeRows;
+ }
+
+ // ======================================================================
+ // High-signal 10-test suite wrappers
+ // ======================================================================
+
+ private static void testFilteringAndTopicSelection() throws Exception {
+ testPathFiltering();
+ testPollWithInfoTopicFilter();
+ }
+
+ private static void testWalCatchUpAndGapRecovery() throws Exception {
+ testBurstWriteGapRecovery();
+ }
+
+ private static void testSeekAndPositionSemantics() throws Exception {
+ testSeekNavigationSemantics();
+ testSeekAfterCheckpointSemantics();
+ testSeekAfterWithStaleAckFencing();
+ }
+
+ private static void testAckNackAndPoisonSemantics() throws Exception {
+ testCommitAfterUnsubscribe();
+ testPoisonMessageDrop();
+ }
+
+ // ======================================================================
+ // Test 8: Consumer Restart Recovery
+ // ======================================================================
+ /**
+ * Verifies:
+ *
+ *
+ * - A committed per-region checkpoint captured by consumer1 can be reused after restart
+ *
- A restarted consumer with the same group can seek to that checkpoint and continue
+ *
- The tail after restart is replayed exactly once
+ *
+ */
+ private static void testConsumerRestartRecovery() throws Exception {
+ String database = nextDatabase();
+ String topicName = nextTopic();
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId1 = nextConsumerId();
+ String consumerId2 = consumerId1 + "_restart";
+ SubscriptionTreePullConsumer consumer1 = null;
+ SubscriptionTreePullConsumer consumer2 = null;
+
+ try {
+ try (ISession session = openSession()) {
+ createDatabase(session, database);
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d1(time, s1) VALUES (0, 0)", database));
+ session.executeNonQueryStatement("flush");
+ }
+ Thread.sleep(2000);
+
+ createTopic(topicName, database + ".**");
+ Thread.sleep(1000);
+
+ consumer1 = createConsumer(consumerId1, consumerGroupId);
+ consumer1.subscribe(topicName);
+ Thread.sleep(3000);
+
+ final int totalRows = 257;
+ System.out.println(" Writing " + totalRows + " rows before restart");
+ try (ISession session = openSession()) {
+ for (int i = 1; i <= totalRows; i++) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d1(time, s1) VALUES (%d, %d)", database, i, i * 10));
+ }
+ session.executeNonQueryStatement("flush");
+ }
+ Thread.sleep(3000);
+
+ SubscriptionMessage committedMessage = null;
+ int committedRows = 0;
+ for (int attempt = 1; attempt <= 30; attempt++) {
+ List messages = consumer1.poll(Duration.ofMillis(2000));
+ if (messages.isEmpty()) {
+ Thread.sleep(1000);
+ continue;
+ }
+ committedMessage = messages.get(0);
+ committedRows = countRows(committedMessage);
+ consumer1.commitSync(committedMessage);
+ break;
+ }
+
+ assertAtLeast("First consumer should commit some rows before restart", 1, committedRows);
+ TopicProgress checkpoint = consumer1.committedPositions(topicName);
+ assertTrue(
+ "Committed checkpoint should not be empty",
+ checkpoint.getRegionProgress() != null && !checkpoint.getRegionProgress().isEmpty());
+ int remainingRows = totalRows - committedRows;
+ assertAtLeast("Restart scenario should leave rows after the first commit", 1, remainingRows);
+ System.out.println(
+ " Captured checkpoint after committing "
+ + committedRows
+ + " rows: "
+ + checkpoint
+ + ", remainingRows="
+ + remainingRows);
+
+ consumer1.close();
+ consumer1 = null;
+
+ consumer2 = createConsumer(consumerId2, consumerGroupId);
+ consumer2.subscribe(topicName);
+ Thread.sleep(3000);
+ consumer2.seekAfter(topicName, checkpoint);
+ Thread.sleep(1000);
+
+ PollResult resumed = pollUntilComplete(consumer2, remainingRows, 120);
+ System.out.println(" Restart recovery result: " + resumed);
+ assertEquals(
+ "Restarted consumer should resume from the committed checkpoint without replay",
+ remainingRows,
+ resumed.totalRows);
+ } finally {
+ cleanup(consumer1, topicName, database);
+ cleanup(consumer2, topicName, database);
+ }
+ }
+
+ // ======================================================================
+ // Test 1: Basic Flow (merged: BasicDataDelivery + MultiDevices + Flush)
+ // ======================================================================
+ /**
+ * Verifies:
+ *
+ *
+ * - Data written BEFORE subscribe is NOT received
+ *
- Multiple devices (d1, d2, d3) written AFTER subscribe are all received
+ *
- Flush does not cause data loss (WAL pinning keeps entries available)
+ *
- Exact row count matches expectation
+ *
+ */
+ private static void testBasicFlow() throws Exception {
+ String database = nextDatabase();
+ String topicName = nextTopic();
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId = nextConsumerId();
+ SubscriptionTreePullConsumer consumer = null;
+
+ try {
+ // Step 1: Write initial data to create DataRegion (should NOT be received)
+ System.out.println(" Step 1: Writing initial data (should NOT be received)");
+ try (ISession session = openSession()) {
+ createDatabase(session, database);
+ for (int i = 0; i < 50; i++) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d1(time, s1) VALUES (%d, %d)", database, i, i * 10));
+ }
+ // Also write to d2, d3 for multi-device readiness
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d2(time, s1) VALUES (0, 0)", database));
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d3(time, s1) VALUES (0, 0)", database));
+ session.executeNonQueryStatement("flush");
+ }
+ Thread.sleep(2000);
+
+ // Step 2: Create topic and subscribe
+ System.out.println(" Step 2: Creating topic and subscribing");
+ createTopic(topicName, database + ".**");
+ Thread.sleep(1000);
+
+ consumer = createConsumer(consumerId, consumerGroupId);
+ consumer.subscribe(topicName);
+ Thread.sleep(3000);
+
+ // Step 3: Write to 3 devices (30 rows each = 90 total), then flush
+ System.out.println(" Step 3: Writing 30 rows x 3 devices AFTER subscribe, then flush");
+ try (ISession session = openSession()) {
+ for (int i = 100; i < 130; i++) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d1(time, s1) VALUES (%d, %d)", database, i, i * 10));
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d2(time, s1) VALUES (%d, %d)", database, i, i * 20));
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d3(time, s1) VALUES (%d, %d)", database, i, i * 30));
+ }
+ System.out.println(" Flushing...");
+ session.executeNonQueryStatement("flush");
+ }
+ Thread.sleep(2000);
+
+ // Step 4: Poll and verify
+ System.out.println(" Step 4: Polling...");
+ PollResult result = pollUntilComplete(consumer, 90, 100);
+ System.out.println(" Result: " + result);
+
+ assertEquals("Expected exactly 90 rows (30 per device)", 90, result.totalRows);
+ if (!result.rowsPerDevice.isEmpty()) {
+ System.out.println(" Rows per device: " + result.rowsPerDevice);
+ for (String dev : new String[] {"d1", "d2", "d3"}) {
+ Integer devRows = result.rowsPerDevice.get(database + "." + dev);
+ assertAtLeast("Expected rows from " + dev, 1, devRows != null ? devRows : 0);
+ }
+ }
+ } finally {
+ cleanup(consumer, topicName, database);
+ }
+ }
+
+ // ======================================================================
+ // Test 2: Data Types (merged: MultipleDataTypes + Aligned + CrossPartition)
+ // ======================================================================
+ /**
+ * Verifies:
+ *
+ *
+ * - Non-aligned: 6 data types (INT32, INT64, FLOAT, DOUBLE, BOOLEAN, TEXT)
+ *
- Aligned: 6 data types, cross-partition timestamps (>1 week apart)
+ *
- 6 write methods: SQL single/multi-row, insertAlignedRecord/Records/Tablet/Tablets
+ *
+ */
+ private static void testDataTypes() throws Exception {
+ String database = nextDatabase();
+ String topicName = nextTopic();
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId = nextConsumerId();
+ SubscriptionTreePullConsumer consumer = null;
+ final long GAP = 604_800_001L; // slightly over 1 week
+
+ try {
+ try (ISession session = openSession()) {
+ createDatabase(session, database);
+ // Create aligned timeseries
+ session.executeNonQueryStatement(
+ String.format(
+ "CREATE ALIGNED TIMESERIES %s.d_aligned"
+ + "(s_int32 INT32, s_int64 INT64, s_float FLOAT,"
+ + " s_double DOUBLE, s_bool BOOLEAN, s_text TEXT)",
+ database));
+ // Init rows to force DataRegion creation
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d1(time, s_int32) VALUES (0, 0)", database));
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO %s.d_aligned(time, s_int32, s_int64, s_float,"
+ + " s_double, s_bool, s_text)"
+ + " VALUES (0, 0, 0, 0.0, 0.0, false, 'init')",
+ database));
+ session.executeNonQueryStatement("flush");
+ }
+ Thread.sleep(2000);
+
+ createTopic(topicName, database + ".**");
+ Thread.sleep(1000);
+
+ consumer = createConsumer(consumerId, consumerGroupId);
+ consumer.subscribe(topicName);
+ Thread.sleep(3000);
+
+ int totalExpected = 0;
+ final String device = database + ".d_aligned";
+ List measurements =
+ Arrays.asList("s_int32", "s_int64", "s_float", "s_double", "s_bool", "s_text");
+ List types =
+ Arrays.asList(
+ TSDataType.INT32,
+ TSDataType.INT64,
+ TSDataType.FLOAT,
+ TSDataType.DOUBLE,
+ TSDataType.BOOLEAN,
+ TSDataType.TEXT);
+ List schemas = new ArrayList<>();
+ schemas.add(new MeasurementSchema("s_int32", TSDataType.INT32));
+ schemas.add(new MeasurementSchema("s_int64", TSDataType.INT64));
+ schemas.add(new MeasurementSchema("s_float", TSDataType.FLOAT));
+ schemas.add(new MeasurementSchema("s_double", TSDataType.DOUBLE));
+ schemas.add(new MeasurementSchema("s_bool", TSDataType.BOOLEAN));
+ schemas.add(new MeasurementSchema("s_text", TSDataType.TEXT));
+
+ try (ISession session = openSession()) {
+ // --- Part A: Non-aligned, 6 types x 20 rows ---
+ System.out.println(" Part A: Non-aligned 6 data types x 20 rows");
+ for (int i = 1; i <= 20; i++) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d1(time, s_int32) VALUES (%d, %d)", database, i, i));
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO %s.d1(time, s_int64) VALUES (%d, %d)",
+ database, i, (long) i * 100000L));
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO %s.d1(time, s_float) VALUES (%d, %f)", database, i, i * 1.1f));
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO %s.d1(time, s_double) VALUES (%d, %f)", database, i, i * 2.2));
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO %s.d1(time, s_bool) VALUES (%d, %s)",
+ database, i, i % 2 == 0 ? "true" : "false"));
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO %s.d1(time, s_text) VALUES (%d, 'text_%d')", database, i, i));
+ }
+ totalExpected += 120; // 6 types x 20 rows
+
+ // --- Part B: Aligned cross-partition, 6 write methods ---
+ System.out.println(" Part B: Aligned cross-partition, 6 write methods");
+
+ // Method 1: SQL single row
+ long t1 = 1;
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO %s.d_aligned(time, s_int32, s_int64, s_float,"
+ + " s_double, s_bool, s_text)"
+ + " VALUES (%d, 1, 100, 1.1, 1.11, true, 'sql_single')",
+ database, t1));
+ totalExpected += 1;
+
+ // Method 2: SQL multi-row (cross-partition)
+ long t2a = 1 + GAP;
+ long t2b = 1 + 2 * GAP;
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO %s.d_aligned(time, s_int32, s_int64, s_float,"
+ + " s_double, s_bool, s_text)"
+ + " VALUES (%d, 2, 200, 2.2, 2.22, false, 'sql_multi_a'),"
+ + " (%d, 3, 300, 3.3, 3.33, true, 'sql_multi_b')",
+ database, t2a, t2b));
+ totalExpected += 2;
+
+ // Method 3: insertAlignedRecord
+ long t3 = 1 + 3 * GAP;
+ session.insertAlignedRecord(
+ device,
+ t3,
+ measurements,
+ types,
+ Arrays.asList(4, 400L, 4.4f, 4.44, true, "record_single"));
+ totalExpected += 1;
+
+ // Method 4: insertAlignedRecordsOfOneDevice (cross-partition)
+ long t4a = 1 + 4 * GAP;
+ long t4b = 1 + 5 * GAP;
+ session.insertAlignedRecordsOfOneDevice(
+ device,
+ Arrays.asList(t4a, t4b),
+ Arrays.asList(measurements, measurements),
+ Arrays.asList(types, types),
+ Arrays.asList(
+ Arrays.asList(5, 500L, 5.5f, 5.55, false, "records_a"),
+ Arrays.asList(6, 600L, 6.6f, 6.66, true, "records_b")));
+ totalExpected += 2;
+
+ // Method 5: insertAlignedTablet (cross-partition)
+ long t5a = 1 + 6 * GAP;
+ long t5b = 1 + 7 * GAP;
+ Tablet tablet5 = new Tablet(device, schemas, 2);
+ addAlignedTabletRow(tablet5, 0, t5a, 7, 700L, 7.7f, 7.77, false, "tablet_a");
+ addAlignedTabletRow(tablet5, 1, t5b, 8, 800L, 8.8f, 8.88, true, "tablet_b");
+ session.insertAlignedTablet(tablet5);
+ totalExpected += 2;
+
+ // Method 6: insertAlignedTablets (cross-partition)
+ long t6a = 1 + 8 * GAP;
+ long t6b = 1 + 9 * GAP;
+ Tablet tablet6 = new Tablet(device, schemas, 2);
+ addAlignedTabletRow(tablet6, 0, t6a, 9, 900L, 9.9f, 9.99, false, "tablets_a");
+ addAlignedTabletRow(tablet6, 1, t6b, 10, 1000L, 10.1f, 10.10, true, "tablets_b");
+ Map tabletMap = new HashMap<>();
+ tabletMap.put(device, tablet6);
+ session.insertAlignedTablets(tabletMap);
+ totalExpected += 2;
+ }
+
+ System.out.println(" Total expected rows: " + totalExpected);
+ Thread.sleep(2000);
+
+ PollResult result = pollUntilComplete(consumer, totalExpected, 150);
+ System.out.println(" Result: " + result);
+
+ assertAtLeast(
+ "Expected at least " + totalExpected + " rows", totalExpected, result.totalRows);
+ assertAtLeast("Expected multiple column types in result", 2, result.seenColumns.size());
+ } finally {
+ cleanup(consumer, topicName, database);
+ }
+ }
+
+ // ======================================================================
+ // Test 3: Path Filtering (merged: DeviceLevel + TimeseriesLevel)
+ // ======================================================================
+ /**
+ * Verifies:
+ *
+ *
+ * - Device-level: topic on d1.** does NOT deliver d2 data
+ *
- Timeseries-level: topic on d1.s1 鈥?lenient check for s2 filtering
+ *
+ */
+ private static void testPathFiltering() throws Exception {
+ String database = nextDatabase();
+ String topicName = nextTopic();
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId = nextConsumerId();
+ SubscriptionTreePullConsumer consumer = null;
+
+ try {
+ try (ISession session = openSession()) {
+ createDatabase(session, database);
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d1(time, s1, s2) VALUES (0, 0, 0)", database));
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d2(time, s1) VALUES (0, 0)", database));
+ session.executeNonQueryStatement("flush");
+ }
+ Thread.sleep(2000);
+
+ // Topic filters d1.s1 only (timeseries-level)
+ String filterPath = database + ".d1.s1";
+ createTopic(topicName, filterPath);
+ Thread.sleep(1000);
+
+ consumer = createConsumer(consumerId, consumerGroupId);
+ consumer.subscribe(topicName);
+ Thread.sleep(3000);
+
+ System.out.println(" Writing to d1 (s1 + s2) and d2 (s1)");
+ try (ISession session = openSession()) {
+ for (int i = 100; i < 150; i++) {
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO %s.d1(time, s1, s2) VALUES (%d, %d, %d)",
+ database, i, i * 10, i * 20));
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d2(time, s1) VALUES (%d, %d)", database, i, i * 30));
+ }
+ }
+ Thread.sleep(2000);
+
+ System.out.println(" Polling (expecting d1 data only, ideally s1 only)...");
+ PollResult result = pollUntilComplete(consumer, 50, 60);
+ System.out.println(" Result: " + result);
+
+ // Device-level: d2 must NOT appear
+ if (!result.rowsPerDevice.isEmpty()) {
+ Integer d2Rows = result.rowsPerDevice.get(database + ".d2");
+ assertTrue("Expected NO rows from d2, but got " + d2Rows, d2Rows == null || d2Rows == 0);
+ Integer d1Rows = result.rowsPerDevice.get(database + ".d1");
+ assertAtLeast("Expected d1 rows", 1, d1Rows != null ? d1Rows : 0);
+ System.out.println(" Device filtering verified: d1=" + d1Rows + ", d2=" + d2Rows);
+ }
+
+ // Timeseries-level: lenient check
+ boolean hasS2 = result.seenColumns.stream().anyMatch(c -> c.contains(".s2"));
+ if (hasS2) {
+ System.out.println(
+ " INFO: Both s1 and s2 received 鈥?converter uses device-level filtering only.");
+ assertAtLeast("Should have received d1 rows", 50, result.totalRows);
+ } else {
+ System.out.println(" Timeseries-level filtering verified: only s1 data received");
+ assertEquals("Expected exactly 50 rows from d1.s1 only", 50, result.totalRows);
+ }
+ } finally {
+ cleanup(consumer, topicName, database);
+ }
+ }
+
+ // ======================================================================
+ // Test 4: Subscribe Before Region Creation (kept as-is)
+ // ======================================================================
+ /**
+ * Subscribe BEFORE the database/region exists, then create database and write. Tests the
+ * IoTConsensus.onNewPeerCreated auto-binding path.
+ */
+ private static void testSubscribeBeforeRegion() throws Exception {
+ String database = nextDatabase();
+ String topicName = nextTopic();
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId = nextConsumerId();
+ SubscriptionTreePullConsumer consumer = null;
+
+ try {
+ System.out.println(" Step 1: Creating topic BEFORE database exists");
+ createTopic(topicName, database + ".**");
+ Thread.sleep(1000);
+
+ System.out.println(" Step 2: Subscribing (no DataRegion exists yet)");
+ consumer = createConsumer(consumerId, consumerGroupId);
+ consumer.subscribe(topicName);
+ Thread.sleep(3000);
+
+ System.out.println(" Step 3: Creating database and writing data (100 rows)");
+ try (ISession session = openSession()) {
+ createDatabase(session, database);
+ for (int i = 0; i < 100; i++) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d1(time, s1) VALUES (%d, %d)", database, i, i * 10));
+ }
+ }
+ Thread.sleep(5000);
+
+ System.out.println(" Step 4: Polling...");
+ PollResult result = pollUntilComplete(consumer, 100, 100);
+ System.out.println(" Result: " + result);
+
+ if (result.totalRows >= 100) {
+ System.out.println(" Auto-binding works! All " + result.totalRows + " rows received.");
+ } else if (result.totalRows > 0) {
+ System.out.println(
+ " Partial: " + result.totalRows + "/100 rows. First writes may precede binding.");
+ } else {
+ System.out.println(" No data received. Check logs for auto-binding messages.");
+ }
+ assertAtLeast(
+ "Expected some rows from subscribe-before-region (auto-binding)", 1, result.totalRows);
+ } finally {
+ cleanup(consumer, topicName, database);
+ }
+ }
+
+ // ======================================================================
+ // Test 6: Multi-Entity Isolation
+ // ======================================================================
+ /**
+ * Verifies:
+ *
+ *
+ * - Two consumer groups on the same topic each receive the full data stream independently
+ *
+ */
+ private static void testMultiEntityIsolation() throws Exception {
+ String database = nextDatabase();
+ String topicName = "topic_multi_" + testCounter;
+ String consumerGroupId1 = "cg_multi_" + testCounter + "_a";
+ String consumerId1 = "consumer_multi_" + testCounter + "_a";
+ String consumerGroupId2 = "cg_multi_" + testCounter + "_b";
+ String consumerId2 = "consumer_multi_" + testCounter + "_b";
+ SubscriptionTreePullConsumer consumer1 = null;
+ SubscriptionTreePullConsumer consumer2 = null;
+
+ try {
+ // Setup: database with a single device path to isolate multi-group semantics.
+ try (ISession session = openSession()) {
+ createDatabase(session, database);
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d1(time, s1) VALUES (0, 0)", database));
+ session.executeNonQueryStatement("flush");
+ }
+ Thread.sleep(2000);
+
+ createTopic(topicName, database + ".d1.**");
+ Thread.sleep(1000);
+
+ consumer1 = createConsumer(consumerId1, consumerGroupId1);
+ consumer1.subscribe(topicName);
+ consumer2 = createConsumer(consumerId2, consumerGroupId2);
+ consumer2.subscribe(topicName);
+ Thread.sleep(3000);
+
+ System.out.println(" Writing 70 rows to d1");
+ try (ISession session = openSession()) {
+ for (int i = 1; i <= 70; i++) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d1(time, s1) VALUES (%d, %d)", database, i, i * 10));
+ }
+ }
+ Thread.sleep(2000);
+
+ System.out.println(" Multi-group isolation");
+ System.out.println(" Polling from group 1...");
+ PollResult result1 = pollUntilComplete(consumer1, 70, 80);
+ System.out.println(" Group 1 result: " + result1);
+
+ System.out.println(" Polling from group 2...");
+ PollResult result2 = pollUntilComplete(consumer2, 70, 80);
+ System.out.println(" Group 2 result: " + result2);
+
+ assertEquals("Group 1 should receive all 70 rows", 70, result1.totalRows);
+ assertEquals("Group 2 should receive all 70 rows", 70, result2.totalRows);
+ assertEquals(
+ "Expected 70 rows from d1", 70, result1.rowsPerDevice.getOrDefault(database + ".d1", 0));
+ assertEquals(
+ "Expected 70 rows from d1", 70, result2.rowsPerDevice.getOrDefault(database + ".d1", 0));
+ System.out.println(
+ " Multi-group isolation verified: group1="
+ + result1.totalRows
+ + ", group2="
+ + result2.totalRows);
+ } finally {
+ if (consumer1 != null) {
+ try {
+ consumer1.unsubscribe(topicName);
+ } catch (Exception e) {
+ /* ignore */
+ }
+ try {
+ consumer1.close();
+ } catch (Exception e) {
+ /* ignore */
+ }
+ }
+ if (consumer2 != null) {
+ try {
+ consumer2.unsubscribe(topicName);
+ } catch (Exception e) {
+ /* ignore */
+ }
+ try {
+ consumer2.close();
+ } catch (Exception e) {
+ /* ignore */
+ }
+ }
+ dropTopic(topicName);
+ deleteDatabase(database);
+ }
+ }
+
+ // ======================================================================
+ // Test 7: Burst Write Gap Recovery (NEW 鈥?tests C2 fix)
+ // ======================================================================
+ /**
+ * Tests that burst writing beyond the pending queue capacity (4096) does not cause data loss. The
+ * pending queue overflow triggers gaps, which should be recovered from WAL.
+ *
+ * Mechanism: Each {@code IoTConsensusServerImpl.write()} call produces exactly one
+ * {@code pendingEntries.offer()}. A single {@code session.insertTablet(tablet)} with N rows in
+ * one time partition = 1 write() call = 1 offer, so Tablet batches rarely overflow the queue. To
+ * actually overflow, we need 4096+ individual write() calls arriving faster than the
+ * prefetch thread can drain. We achieve this with multiple concurrent writer threads, each
+ * performing individual SQL INSERTs, to maximize the aggregate write rate vs. drain rate.
+ *
+ *
Note: Gap occurrence is inherently timing-dependent (race between writers and the
+ * prefetch drain loop). This test maximizes the probability by using concurrent threads, but
+ * cannot guarantee gap occurrence on every run. Check server logs for "gap detected" / "Filling
+ * from WAL" messages to confirm the gap path was exercised.
+ *
+ *
Fix verified: C2 鈥?gap entries are not skipped when WAL fill times out; they are deferred to
+ * the next prefetch iteration.
+ */
+ private static void testBurstWriteGapRecovery() throws Exception {
+ String database = nextDatabase();
+ String topicName = nextTopic();
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId = nextConsumerId();
+ SubscriptionTreePullConsumer consumer = null;
+
+ try {
+ try (ISession session = openSession()) {
+ createDatabase(session, database);
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d1(time, s1) VALUES (0, 0)", database));
+ session.executeNonQueryStatement("flush");
+ }
+ Thread.sleep(2000);
+
+ createTopic(topicName, database + ".**");
+ Thread.sleep(1000);
+
+ consumer = createConsumer(consumerId, consumerGroupId);
+ consumer.subscribe(topicName);
+ Thread.sleep(3000);
+
+ // Use multiple concurrent writer threads with individual SQL INSERTs.
+ // Each INSERT 鈫?1 IoTConsensusServerImpl.write() 鈫?1 pendingEntries.offer().
+ // With N threads writing concurrently, aggregate rate should exceed drain rate
+ // and overflow the 4096-capacity queue, creating gaps.
+ final int writerThreads = 4;
+ final int rowsPerThread = 1500; // 4 * 1500 = 6000 total write() calls > 4096
+ final int totalRows = writerThreads * rowsPerThread;
+ final AtomicInteger errorCount = new AtomicInteger(0);
+ final CountDownLatch startLatch = new CountDownLatch(1);
+ final CountDownLatch doneLatch = new CountDownLatch(writerThreads);
+
+ System.out.println(
+ " Burst writing "
+ + totalRows
+ + " rows via "
+ + writerThreads
+ + " concurrent threads ("
+ + rowsPerThread
+ + " individual SQL INSERTs each)");
+ System.out.println(
+ " (Each INSERT = 1 WAL entry = 1 pendingEntries.offer(); " + "queue capacity = 4096)");
+
+ ExecutorService executor = Executors.newFixedThreadPool(writerThreads);
+ for (int t = 0; t < writerThreads; t++) {
+ final int threadId = t;
+ final int startTs = threadId * rowsPerThread + 1;
+ executor.submit(
+ () -> {
+ try {
+ startLatch.await(); // all threads start at the same time
+ try (ISession session = openSession()) {
+ for (int i = 0; i < rowsPerThread; i++) {
+ int ts = startTs + i;
+ session.executeNonQueryStatement(
+ String.format(
+ "INSERT INTO %s.d1(time, s1) VALUES (%d, %d)",
+ database, ts, (long) ts * 10));
+ }
+ }
+ } catch (Exception e) {
+ System.out.println(" Writer thread " + threadId + " error: " + e.getMessage());
+ errorCount.incrementAndGet();
+ } finally {
+ doneLatch.countDown();
+ }
+ });
+ }
+
+ // Fire all threads simultaneously
+ startLatch.countDown();
+ doneLatch.await();
+ executor.shutdown();
+
+ if (errorCount.get() > 0) {
+ System.out.println(" WARNING: " + errorCount.get() + " writer threads encountered errors");
+ }
+
+ // Do NOT add artificial delay 鈥?let the consumer compete with ongoing WAL writes
+ System.out.println(
+ " Polling (expecting " + totalRows + " rows, may need WAL gap recovery)...");
+ System.out.println(
+ " (Check server logs for 'gap detected' to confirm gap recovery was triggered)");
+ PollResult result = pollUntilComplete(consumer, totalRows, 6000, 2000, true);
+ System.out.println(" Result: " + result);
+
+ assertEquals(
+ "Expected exactly " + totalRows + " rows (no data loss despite pending queue overflow)",
+ totalRows,
+ result.totalRows);
+ } finally {
+ cleanup(consumer, topicName, database);
+ }
+ }
+
+ // ======================================================================
+ // Test 8: Commit After Unsubscribe (NEW 鈥?tests H7 fix)
+ // ======================================================================
+ /**
+ * Tests that commit still works correctly after the consumer has unsubscribed (queue has been
+ * torn down). The commit routing should use metadata-based topic config check instead of runtime
+ * queue state.
+ *
+ *
Fix verified: H7 鈥?commit routes via isConsensusBasedTopic() instead of hasQueue().
+ */
+ private static void testCommitAfterUnsubscribe() throws Exception {
+ String database = nextDatabase();
+ String topicName = nextTopic();
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId = nextConsumerId();
+ SubscriptionTreePullConsumer consumer = null;
+
+ try {
+ try (ISession session = openSession()) {
+ createDatabase(session, database);
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d1(time, s1) VALUES (0, 0)", database));
+ session.executeNonQueryStatement("flush");
+ }
+ Thread.sleep(2000);
+
+ createTopic(topicName, database + ".**");
+ Thread.sleep(1000);
+
+ consumer = createConsumer(consumerId, consumerGroupId);
+ consumer.subscribe(topicName);
+ Thread.sleep(3000);
+
+ // Write data
+ System.out.println(" Writing 50 rows");
+ try (ISession session = openSession()) {
+ for (int i = 1; i <= 50; i++) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d1(time, s1) VALUES (%d, %d)", database, i, i * 10));
+ }
+ }
+ Thread.sleep(2000);
+
+ // Poll WITHOUT commit
+ System.out.println(" Polling WITHOUT commit...");
+ List uncommittedMessages = new ArrayList<>();
+ int polledRows = 0;
+ for (int attempt = 0; attempt < 60 && polledRows < 50; attempt++) {
+ List msgs = consumer.poll(Duration.ofMillis(2000));
+ if (msgs.isEmpty()) {
+ if (polledRows > 0) break;
+ Thread.sleep(500);
+ continue;
+ }
+ for (SubscriptionMessage msg : msgs) {
+ uncommittedMessages.add(msg);
+ for (SubscriptionResultSet ds : getResultSets(msg)) {
+ while (ds.hasNext()) {
+ ds.next();
+ polledRows++;
+ }
+ }
+ }
+ }
+ System.out.println(
+ " Polled "
+ + polledRows
+ + " rows, holding "
+ + uncommittedMessages.size()
+ + " uncommitted messages");
+ assertAtLeast("Should have polled some rows before unsubscribe", 1, polledRows);
+
+ // Unsubscribe (tears down the consensus queue)
+ System.out.println(" Unsubscribing (queue teardown)...");
+ consumer.unsubscribe(topicName);
+ Thread.sleep(2000);
+
+ // Now commit the previously polled messages 鈥?should NOT throw
+ System.out.println(
+ " Committing " + uncommittedMessages.size() + " messages AFTER unsubscribe...");
+ boolean commitSucceeded = true;
+ for (SubscriptionMessage msg : uncommittedMessages) {
+ try {
+ consumer.commitSync(msg);
+ } catch (Exception e) {
+ System.out.println(" Commit threw exception: " + e.getMessage());
+ commitSucceeded = false;
+ }
+ }
+
+ // The commit may silently succeed or fail gracefully 鈥?the key is no crash
+ System.out.println(" Commit after unsubscribe completed. Success=" + commitSucceeded);
+ assertTrue("Commit after unsubscribe should succeed without exception", commitSucceeded);
+ System.out.println(" (Key: no exception crash, routing handled gracefully)");
+ } finally {
+ if (consumer != null) {
+ try {
+ consumer.close();
+ } catch (Exception e) {
+ /* ignore */
+ }
+ }
+ dropTopic(topicName);
+ deleteDatabase(database);
+ }
+ }
+
+ /**
+ * Verifies:
+ *
+ *
+ * - seekToBeginning replays historical rows from the beginning of the topic
+ *
- seekToEnd suppresses old rows and only delivers future writes
+ *
- seek(topicProgress) resumes from a committed checkpoint without replaying earlier rows
+ *
+ */
+ private static void testSeekNavigationSemantics() throws Exception {
+ String database = nextDatabase();
+ String topicName = nextTopic();
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId = nextConsumerId();
+ SubscriptionTreePullConsumer consumer = null;
+
+ final int initialRows = 1800;
+ final int rowsAfterSeekToEnd = 240;
+
+ try {
+ bootstrapSeekTopic(database, topicName);
+ consumer = createSubscribedConsumer(topicName, consumerId, consumerGroupId);
+
+ System.out.println(" Step 1: Write initial live rows and drain them");
+ writeSequentialRowsAndFlush(database, 1000, initialRows);
+ PollResult firstPoll = pollUntilComplete(consumer, initialRows, 120);
+ System.out.println(" First poll: " + firstPoll.totalRows + " rows");
+ assertEquals(
+ "Initial live poll should deliver exactly the rows written after subscribe",
+ initialRows,
+ firstPoll.totalRows);
+
+ System.out.println(" Step 2: seekToBeginning -> expect full replay");
+ consumer.seekToBeginning(topicName);
+ pause(2000);
+
+ PollResult beginningPoll = pollUntilComplete(consumer, initialRows, 120);
+ System.out.println(" After seekToBeginning: " + beginningPoll.totalRows + " rows");
+ assertAtLeast(
+ "seekToBeginning should replay the rows written after subscribe",
+ initialRows,
+ beginningPoll.totalRows);
+ assertAtMost(
+ "seekToBeginning should replay at most one extra bootstrap row",
+ initialRows + 1,
+ beginningPoll.totalRows);
+
+ System.out.println(" Step 3: seekToEnd -> expect no old data");
+ consumer.seekToEnd(topicName);
+ pause(2000);
+
+ PollResult endPoll = pollUntilComplete(consumer, 0, 15, 1000, true);
+ System.out.println(" After seekToEnd with no new writes: " + endPoll.totalRows + " rows");
+ assertAtMost("seekToEnd should yield at most 1 race row", 1, endPoll.totalRows);
+
+ System.out.println(" Step 4: Write new rows after seekToEnd");
+ writeSequentialRowsAndFlush(database, 4000, rowsAfterSeekToEnd);
+ PollResult afterEndPoll = pollUntilComplete(consumer, rowsAfterSeekToEnd, 120);
+ System.out.println(" After seekToEnd + new writes: " + afterEndPoll.totalRows + " rows");
+ assertEquals(
+ "seekToEnd should only deliver rows written after the seek",
+ rowsAfterSeekToEnd,
+ afterEndPoll.totalRows);
+
+ System.out.println(" Step 5: seek(committed checkpoint) -> expect remaining tail only");
+ consumer.seekToBeginning(topicName);
+ pause(2000);
+
+ CommittedSnapshot midpointCheckpoint =
+ pollUntilCommittedRows(consumer, topicName, initialRows / 2, 60, 1000);
+ List remainingTail =
+ pollAndCaptureCommittedSnapshots(consumer, topicName, 60, 1000);
+ int expectedRemainingRows = totalRows(remainingTail);
+ System.out.println(
+ " Midpoint checkpoint after "
+ + midpointCheckpoint.cumulativeRows
+ + " rows, expected remaining tail="
+ + expectedRemainingRows);
+ assertAtLeast(
+ "seek(topicProgress) scenario should leave rows after the checkpoint",
+ 1,
+ expectedRemainingRows);
+
+ consumer.seekAfter(topicName, midpointCheckpoint.progress);
+ pause(2000);
+
+ PollResult afterSeek = pollUntilComplete(consumer, expectedRemainingRows, 120);
+ System.out.println(" After seek(topicProgress): " + afterSeek.totalRows + " rows");
+ assertEquals(
+ "seek(topicProgress) should resume from the committed checkpoint",
+ expectedRemainingRows,
+ afterSeek.totalRows);
+ } finally {
+ cleanup(consumer, topicName, database);
+ }
+ }
+
+ /**
+ * Verifies:
+ *
+ *
+ * - seekAfter(topicProgress) replays only rows strictly after the checkpoint
+ *
- Repeating seekAfter with the same checkpoint is stable
+ *
- seekAfter(tail) suppresses history but still allows future rows through
+ *
+ */
+ private static void testSeekAfterCheckpointSemantics() throws Exception {
+ String database = nextDatabase();
+ String topicName = nextTopic();
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId = nextConsumerId();
+ SubscriptionTreePullConsumer consumer = null;
+
+ final int totalRows = 2000;
+ final int futureRows = 160;
+
+ try {
+ bootstrapSeekTopic(database, topicName);
+ consumer = createSubscribedConsumer(topicName, consumerId, consumerGroupId);
+
+ System.out.println(" Step 1: Write rows and capture a committed checkpoint");
+ writeSequentialRowsAndFlush(database, 1000, totalRows);
+ CommittedSnapshot checkpoint =
+ pollUntilCommittedRows(consumer, topicName, totalRows / 3, 60, 1000);
+ List drainedTail =
+ pollAndCaptureCommittedSnapshots(consumer, topicName, 80, 1000);
+ int expectedTailRows = totalRows(drainedTail);
+ System.out.println(
+ " Checkpoint after "
+ + checkpoint.cumulativeRows
+ + " rows, tail after checkpoint="
+ + expectedTailRows);
+ assertAtLeast(
+ "seekAfter(topicProgress) scenario should leave rows after the checkpoint",
+ 1,
+ expectedTailRows);
+
+ int writerFrontierCount = countWriterFrontiers(checkpoint.progress);
+ assertAtLeast("Committed checkpoint should contain writer frontiers", 1, writerFrontierCount);
+
+ System.out.println(" Step 2: seekAfter(midpoint checkpoint) -> expect exact tail replay");
+ consumer.seekAfter(topicName, checkpoint.progress);
+ pause(2000);
+
+ PollResult firstReplay = pollUntilComplete(consumer, expectedTailRows, 120);
+ System.out.println(" After first seekAfter(checkpoint): " + firstReplay.totalRows + " rows");
+ assertEquals(
+ "seekAfter(topicProgress) should replay exactly the tail after the checkpoint",
+ expectedTailRows,
+ firstReplay.totalRows);
+
+ System.out.println(" Step 3: repeat seekAfter(checkpoint) -> expect same exact replay");
+ consumer.seekAfter(topicName, checkpoint.progress);
+ pause(2000);
+
+ PollResult repeatedReplay = pollUntilComplete(consumer, expectedTailRows, 120);
+ System.out.println(
+ " After repeated seekAfter(checkpoint): " + repeatedReplay.totalRows + " rows");
+ assertEquals(
+ "Repeating seekAfter(topicProgress) should be stable",
+ expectedTailRows,
+ repeatedReplay.totalRows);
+
+ System.out.println(" Step 4: seekAfter(tail) -> expect no historical rows");
+ TopicProgress tailProgress = consumer.committedPositions(topicName);
+ assertTrue("Tail checkpoint should be non-null", tailProgress != null);
+ consumer.seekAfter(topicName, tailProgress);
+ pause(2000);
+
+ PollResult noHistory = pollUntilComplete(consumer, 0, 15, 1000, true);
+ System.out.println(" After seekAfter(tail): " + noHistory.totalRows + " rows");
+ assertAtMost("seekAfter(tail) should yield at most 1 race row", 1, noHistory.totalRows);
+
+ System.out.println(" Step 5: Write new rows after seekAfter(tail)");
+ writeSequentialRowsAndFlush(database, 5000, futureRows);
+ PollResult futureOnly = pollUntilComplete(consumer, futureRows, 120);
+ System.out.println(" After seekAfter(tail) + new writes: " + futureOnly.totalRows + " rows");
+ assertEquals(
+ "seekAfter(tail) should only deliver rows written after the seek",
+ futureRows,
+ futureOnly.totalRows);
+ } finally {
+ cleanup(consumer, topicName, database);
+ }
+ }
+
+ /**
+ * Verifies:
+ *
+ *
+ * - seekAfter fences off stale in-flight commit contexts from before the seek
+ *
- Committing old polled messages after the seek does not affect the new replay frontier
+ *
- The full tail after the checkpoint is replayed exactly once
+ *
+ */
+ private static void testSeekAfterWithStaleAckFencing() throws Exception {
+ String database = nextDatabase();
+ String topicName = nextTopic();
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId = nextConsumerId();
+ SubscriptionTreePullConsumer consumer = null;
+
+ final int totalRows = 2400;
+
+ try {
+ bootstrapSeekTopic(database, topicName);
+ consumer = createSubscribedConsumer(topicName, consumerId, consumerGroupId);
+
+ System.out.println(" Step 1: Write rows and commit part of them");
+ writeSequentialRowsAndFlush(database, 1000, totalRows);
+ CommittedSnapshot committedCheckpoint =
+ pollUntilCommittedRows(consumer, topicName, totalRows / 3, 60, 1000);
+ assertTrue(
+ "Committed checkpoint should not be null",
+ committedCheckpoint.progress != null
+ && committedCheckpoint.progress.getRegionProgress() != null
+ && !committedCheckpoint.progress.getRegionProgress().isEmpty());
+
+ System.out.println(" Step 2: Poll a stale batch without committing it");
+ PolledMessageBatch staleBatch = pollFirstNonEmptyBatchWithoutCommit(consumer, 30, 1000);
+ assertAtLeast(
+ "Stale-ack scenario should poll at least one row after the checkpoint",
+ 1,
+ staleBatch.totalRows);
+
+ int expectedTailRows = totalRows - committedCheckpoint.cumulativeRows;
+ System.out.println(
+ " Committed checkpoint after "
+ + committedCheckpoint.cumulativeRows
+ + " rows, stale batch="
+ + staleBatch.totalRows
+ + ", expected replay tail="
+ + expectedTailRows);
+ assertAtLeast(
+ "Stale-ack replay should include the stale batch rows",
+ staleBatch.totalRows,
+ expectedTailRows);
+
+ System.out.println(" Step 3: seekAfter(checkpoint), then commit stale messages");
+ consumer.seekAfter(topicName, committedCheckpoint.progress);
+ pause(2000);
+
+ for (SubscriptionMessage staleMessage : staleBatch.messages) {
+ consumer.commitSync(staleMessage);
+ }
+
+ PollResult replayAfterSeek = pollUntilComplete(consumer, expectedTailRows, 120);
+ System.out.println(
+ " After seekAfter(checkpoint) with stale commits: "
+ + replayAfterSeek.totalRows
+ + " rows");
+ assertEquals(
+ "Stale commits from the old generation must not reduce the replayed tail",
+ expectedTailRows,
+ replayAfterSeek.totalRows);
+ } finally {
+ cleanup(consumer, topicName, database);
+ }
+ }
+
+ // ======================================================================
+ // Test 11: pollWithInfo(topicNames, timeoutMs) 鈥?topic-level filtering
+ // ======================================================================
+ /**
+ * Verifies:
+ *
+ *
+ * - pollWithInfo(Set, long) only returns data matching the specified topics
+ *
- Data from other subscribed topics is not returned in the filtered poll
+ *
- After filtered poll, remaining data can still be retrieved via unfiltered poll
+ *
+ */
+ private static void testPollWithInfoTopicFilter() throws Exception {
+ String database = nextDatabase();
+ String topicName1 = "topic_pwf_" + testCounter + "_a";
+ String topicName2 = "topic_pwf_" + testCounter + "_b";
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId = nextConsumerId();
+ SubscriptionTreePullConsumer consumer = null;
+
+ try {
+ // Step 0: Create database with d1, d2
+ try (ISession session = openSession()) {
+ createDatabase(session, database);
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d1(time, s1) VALUES (0, 0)", database));
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d2(time, s1) VALUES (0, 0)", database));
+ session.executeNonQueryStatement("flush");
+ }
+ Thread.sleep(2000);
+
+ // Step 1: Create two topics with distinct path filters
+ System.out.println(" Step 1: Creating two topics (d1 / d2)");
+ createTopic(topicName1, database + ".d1.**");
+ createTopic(topicName2, database + ".d2.**");
+ Thread.sleep(1000);
+
+ // Step 2: Subscribe to both topics
+ consumer = createConsumer(consumerId, consumerGroupId);
+ consumer.subscribe(topicName1, topicName2);
+ Thread.sleep(3000);
+
+ // Step 3: Write 30 rows to d1, 40 rows to d2
+ System.out.println(" Step 3: Writing 30 rows to d1, 40 rows to d2");
+ try (ISession session = openSession()) {
+ for (int i = 1; i <= 40; i++) {
+ if (i <= 30) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d1(time, s1) VALUES (%d, %d)", database, i, i * 10));
+ }
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d2(time, s1) VALUES (%d, %d)", database, i, i * 20));
+ }
+ }
+ Thread.sleep(3000);
+
+ // Step 4: pollWithInfo for topicName1 only
+ System.out.println(" Step 4: pollWithInfo for topic1 (d1) only");
+ Set topic1Only = new HashSet<>(Arrays.asList(topicName1));
+ int d1Rows = 0;
+ for (int attempt = 0; attempt < 40; attempt++) {
+ org.apache.iotdb.session.subscription.payload.PollResult pollResult =
+ consumer.pollWithInfo(topic1Only, 2000);
+ List msgs = pollResult.getMessages();
+ if (msgs.isEmpty()) {
+ if (d1Rows > 0) break;
+ Thread.sleep(1000);
+ continue;
+ }
+ for (SubscriptionMessage msg : msgs) {
+ for (SubscriptionResultSet ds : getResultSets(msg)) {
+ List cols = ds.getColumnNames();
+ while (ds.hasNext()) {
+ ds.next();
+ d1Rows++;
+ // Verify no d2 columns appear
+ for (String col : cols) {
+ assertTrue("Topic1 poll should not contain d2 data: " + col, !col.contains(".d2."));
+ }
+ }
+ }
+ consumer.commitSync(msg);
+ }
+ }
+ System.out.println(" Topic1-only poll received: " + d1Rows + " rows");
+ assertEquals("Topic1 should deliver exactly 30 rows from d1", 30, d1Rows);
+
+ // Step 5: pollWithInfo for topicName2 only 鈥?should get d2 data
+ System.out.println(" Step 5: pollWithInfo for topic2 (d2) only");
+ Set topic2Only = new HashSet<>(Arrays.asList(topicName2));
+ int d2Rows = 0;
+ for (int attempt = 0; attempt < 40; attempt++) {
+ org.apache.iotdb.session.subscription.payload.PollResult pollResult =
+ consumer.pollWithInfo(topic2Only, 2000);
+ List msgs = pollResult.getMessages();
+ if (msgs.isEmpty()) {
+ if (d2Rows > 0) break;
+ Thread.sleep(1000);
+ continue;
+ }
+ for (SubscriptionMessage msg : msgs) {
+ for (SubscriptionResultSet ds : getResultSets(msg)) {
+ List cols = ds.getColumnNames();
+ while (ds.hasNext()) {
+ ds.next();
+ d2Rows++;
+ // Verify no d1 columns appear
+ for (String col : cols) {
+ assertTrue("Topic2 poll should not contain d1 data: " + col, !col.contains(".d1."));
+ }
+ }
+ }
+ consumer.commitSync(msg);
+ }
+ }
+ System.out.println(" Topic2-only poll received: " + d2Rows + " rows");
+ assertEquals("Topic2 should deliver exactly 40 rows from d2", 40, d2Rows);
+
+ System.out.println(" testPollWithInfoTopicFilter passed!");
+ } finally {
+ if (consumer != null) {
+ try {
+ consumer.unsubscribe(topicName1, topicName2);
+ } catch (Exception e) {
+ /* ignore */
+ }
+ try {
+ consumer.close();
+ } catch (Exception e) {
+ /* ignore */
+ }
+ }
+ dropTopic(topicName1);
+ dropTopic(topicName2);
+ deleteDatabase(database);
+ }
+ }
+
+ // ======================================================================
+ // Test 12: Poison Message Drop 鈥?messages nacked beyond threshold
+ // are force-acked (dropped) and don't block new data.
+ // ======================================================================
+ /**
+ * Verifies:
+ *
+ *
+ * - A message that is nacked (poll timeout without commit) more than
+ * POISON_MESSAGE_NACK_THRESHOLD (10) times is eventually dropped
+ *
- After the poison message is dropped, new data can still be received
+ *
- The consumer is not permanently blocked by a single unprocessable message
+ *
+ *
+ * Note: "Nack" in this context means the server re-enqueues an in-flight event that was
+ * polled but never committed by the consumer. Each re-enqueue increments the event's nack
+ * counter. After 10 nacks, the event is marked as poisoned and force-acked (dropped) at the next
+ * re-enqueue attempt.
+ */
+ private static void testPoisonMessageDrop() throws Exception {
+ String database = nextDatabase();
+ String topicName = nextTopic();
+ String consumerGroupId = nextConsumerGroup();
+ String consumerId = nextConsumerId();
+ SubscriptionTreePullConsumer consumer = null;
+
+ try {
+ // Step 0: Create DataRegion
+ try (ISession session = openSession()) {
+ createDatabase(session, database);
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d1(time, s1) VALUES (0, 0)", database));
+ session.executeNonQueryStatement("flush");
+ }
+ Thread.sleep(2000);
+
+ // Step 1: Create topic and subscribe
+ System.out.println(" Step 1: Creating topic and subscribing");
+ createTopic(topicName, database + ".**");
+ Thread.sleep(1000);
+
+ consumer = createConsumer(consumerId, consumerGroupId);
+ consumer.subscribe(topicName);
+ Thread.sleep(3000);
+
+ // Step 2: Write initial data that will become the "poison" message
+ System.out.println(" Step 2: Writing 10 rows (the initial batch)");
+ try (ISession session = openSession()) {
+ for (int i = 1; i <= 10; i++) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d1(time, s1) VALUES (%d, %d)", database, i, i * 10));
+ }
+ }
+ Thread.sleep(2000);
+
+ // Step 3: Poll without commit 鈥?repeatedly. Each poll-then-timeout cycle
+ // causes the server to nack the in-flight event and re-enqueue it.
+ // After POISON_MESSAGE_NACK_THRESHOLD (10) nacks, the message should be dropped.
+ System.out.println(
+ " Step 3: Polling without commit for 15 rounds (threshold=10, need >10 nacks)");
+ int totalPoisonPolled = 0;
+ for (int round = 1; round <= 15; round++) {
+ List msgs = consumer.poll(Duration.ofMillis(3000));
+ int roundRows = 0;
+ for (SubscriptionMessage msg : msgs) {
+ for (SubscriptionResultSet ds : getResultSets(msg)) {
+ while (ds.hasNext()) {
+ ds.next();
+ roundRows++;
+ totalPoisonPolled++;
+ }
+ }
+ // Deliberately NOT committing 鈥?this is the "nack" behavior
+ }
+ System.out.println(
+ " Round " + round + ": received " + roundRows + " rows (NOT committing)");
+ if (msgs.isEmpty() && round > 11) {
+ // After threshold exceeded, the message may have been dropped
+ System.out.println(" No messages 鈥?poison message may have been force-acked");
+ break;
+ }
+ Thread.sleep(1000);
+ }
+ System.out.println(" Total rows polled across all rounds: " + totalPoisonPolled);
+
+ // Step 4: Write NEW data and verify it can be received (consumer not blocked)
+ System.out.println(" Step 4: Writing 50 NEW rows and polling WITH commit");
+ try (ISession session = openSession()) {
+ for (int i = 1000; i < 1050; i++) {
+ session.executeNonQueryStatement(
+ String.format("INSERT INTO %s.d1(time, s1) VALUES (%d, %d)", database, i, i * 10));
+ }
+ }
+ Thread.sleep(2000);
+
+ PollResult newResult = pollUntilComplete(consumer, 50, 60);
+ System.out.println(" New data poll result: " + newResult);
+
+ // The key assertion: new data must be receivable
+ // The exact count may be slightly more than 50 if the old poison data leaked through
+ // in an earlier round, but the queue must not be permanently blocked.
+ assertAtLeast(
+ "Consumer must not be permanently blocked by poison message 鈥?new data should arrive",
+ 1,
+ newResult.totalRows);
+ System.out.println(
+ " testPoisonMessageDrop passed: consumer received "
+ + newResult.totalRows
+ + " new rows after poison message handling");
+ } finally {
+ cleanup(consumer, topicName, database);
+ }
+ }
+
+ private static List getResultSets(final SubscriptionMessage message) {
+ return message.getResultSets().stream()
+ .map(resultSet -> (SubscriptionResultSet) resultSet)
+ .collect(Collectors.toList());
+ }
+
+ /** Helper: populate one row of an aligned Tablet with all 6 data types. */
+ private static void addAlignedTabletRow(
+ Tablet tablet,
+ int rowIndex,
+ long timestamp,
+ int intVal,
+ long longVal,
+ float floatVal,
+ double doubleVal,
+ boolean boolVal,
+ String textVal) {
+ tablet.addTimestamp(rowIndex, timestamp);
+ tablet.addValue("s_int32", rowIndex, intVal);
+ tablet.addValue("s_int64", rowIndex, longVal);
+ tablet.addValue("s_float", rowIndex, floatVal);
+ tablet.addValue("s_double", rowIndex, doubleVal);
+ tablet.addValue("s_bool", rowIndex, boolVal);
+ tablet.addValue("s_text", rowIndex, new Binary(textVal, TSFileConfig.STRING_CHARSET));
+ }
+}
diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
index 53c23626b1335..761bae4bd98ef 100644
--- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
+++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
@@ -313,6 +313,7 @@ public enum TSStatusCode {
SHOW_SUBSCRIPTION_ERROR(1910),
SUBSCRIPTION_PIPE_TIMEOUT_ERROR(1911),
SUBSCRIPTION_NOT_ENABLED_ERROR(1912),
+ SUBSCRIPTION_SEEK_ERROR(1913),
// Topic
CREATE_TOPIC_ERROR(2000),
diff --git a/iotdb-client/subscription/pom.xml b/iotdb-client/subscription/pom.xml
index c41ef1e3bde89..bd1e71232b307 100644
--- a/iotdb-client/subscription/pom.xml
+++ b/iotdb-client/subscription/pom.xml
@@ -77,5 +77,10 @@
org.apache.thrift
libthrift
+
+ junit
+ junit
+ test
+
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java
index ea588f1276325..b6c318d6e7628 100644
--- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConfig.java
@@ -35,6 +35,16 @@
public class TopicConfig extends PipeParameters {
+ private static final Set ORDER_MODE_VALUE_SET;
+
+ static {
+ final Set orderModes = new HashSet<>(3);
+ orderModes.add(TopicConstant.ORDER_MODE_LEADER_ONLY_VALUE);
+ orderModes.add(TopicConstant.ORDER_MODE_MULTI_WRITER_VALUE);
+ orderModes.add(TopicConstant.ORDER_MODE_PER_WRITER_VALUE);
+ ORDER_MODE_VALUE_SET = Collections.unmodifiableSet(orderModes);
+ }
+
public TopicConfig() {
super(Collections.emptyMap());
}
@@ -97,6 +107,22 @@ public boolean isTableTopic() {
attributes.getOrDefault(SQL_DIALECT_KEY, SQL_DIALECT_TREE_VALUE));
}
+ public String getOrderMode() {
+ return normalizeOrderMode(
+ attributes.getOrDefault(
+ TopicConstant.ORDER_MODE_KEY, TopicConstant.ORDER_MODE_DEFAULT_VALUE));
+ }
+
+ public static boolean isValidOrderMode(final String orderMode) {
+ return ORDER_MODE_VALUE_SET.contains(normalizeOrderMode(orderMode));
+ }
+
+ public static String normalizeOrderMode(final String orderMode) {
+ return orderMode == null
+ ? TopicConstant.ORDER_MODE_DEFAULT_VALUE
+ : orderMode.trim().toLowerCase();
+ }
+
/////////////////////////////// extractor attributes mapping ///////////////////////////////
public Map getAttributeWithSqlDialect() {
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java
index bb84358648e59..09a63c939bc86 100644
--- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/TopicConstant.java
@@ -40,6 +40,12 @@ public class TopicConstant {
public static final String MODE_SNAPSHOT_VALUE = "snapshot";
public static final String MODE_DEFAULT_VALUE = MODE_LIVE_VALUE;
+ public static final String ORDER_MODE_KEY = "order-mode";
+ public static final String ORDER_MODE_LEADER_ONLY_VALUE = "leader-only";
+ public static final String ORDER_MODE_MULTI_WRITER_VALUE = "multi-writer";
+ public static final String ORDER_MODE_PER_WRITER_VALUE = "per-writer";
+ public static final String ORDER_MODE_DEFAULT_VALUE = ORDER_MODE_LEADER_ONLY_VALUE;
+
public static final String FORMAT_KEY = "format";
public static final String FORMAT_RECORD_HANDLER_VALUE = "SubscriptionRecordHandler";
public static final String FORMAT_TS_FILE_VALUE = "SubscriptionTsFileHandler";
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/RegionProgress.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/RegionProgress.java
new file mode 100644
index 0000000000000..134f59dfe5dae
--- /dev/null
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/RegionProgress.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.rpc.subscription.payload.poll;
+
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Objects;
+
+public class RegionProgress {
+
+ private final Map writerPositions;
+
+ public RegionProgress(final Map writerPositions) {
+ this.writerPositions =
+ writerPositions == null
+ ? Collections.emptyMap()
+ : Collections.unmodifiableMap(new LinkedHashMap<>(writerPositions));
+ }
+
+ public Map getWriterPositions() {
+ return writerPositions;
+ }
+
+ public void serialize(final DataOutputStream stream) throws IOException {
+ ReadWriteIOUtils.write(writerPositions.size(), stream);
+ for (final Map.Entry entry : writerPositions.entrySet()) {
+ entry.getKey().serialize(stream);
+ entry.getValue().serialize(stream);
+ }
+ }
+
+ public static RegionProgress deserialize(final ByteBuffer buffer) {
+ final int size = ReadWriteIOUtils.readInt(buffer);
+ final Map writerPositions = new LinkedHashMap<>(size);
+ for (int i = 0; i < size; i++) {
+ writerPositions.put(WriterId.deserialize(buffer), WriterProgress.deserialize(buffer));
+ }
+ return new RegionProgress(writerPositions);
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof RegionProgress)) {
+ return false;
+ }
+ final RegionProgress that = (RegionProgress) obj;
+ return Objects.equals(writerPositions, that.writerPositions);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(writerPositions);
+ }
+
+ @Override
+ public String toString() {
+ return "RegionProgress{" + "writerPositions=" + writerPositions + '}';
+ }
+}
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionCommitContext.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionCommitContext.java
index e2bf809d32c20..3121843e62c92 100644
--- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionCommitContext.java
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionCommitContext.java
@@ -30,6 +30,13 @@
public class SubscriptionCommitContext implements Comparable {
+ /**
+ * Version 1: original 5 fields (dataNodeId, rebootTimes, topicName, consumerGroupId, commitId).
+ * Version 2: added regionId + physicalTime (serialized in the legacy epoch slot). Version 3:
+ * added writerId + writerProgress.
+ */
+ private static final byte SERIALIZATION_VERSION = 3;
+
private final int dataNodeId;
private final int rebootTimes;
@@ -40,6 +47,16 @@ public class SubscriptionCommitContext implements Comparable
+ Objects.nonNull(context.getWriterId()) ? context.getWriterId().getNodeId() : -1)
+ .thenComparingLong(
+ context ->
+ Objects.nonNull(context.getWriterId())
+ ? context.getWriterId().getWriterEpoch()
+ : -1L)
+ .thenComparingLong(SubscriptionCommitContext::getPhysicalTime)
+ .thenComparingLong(SubscriptionCommitContext::getLocalSeq)
.compare(this, that);
}
}
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollRequest.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollRequest.java
index 3337887b185f5..d8c800f247b2d 100644
--- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollRequest.java
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollRequest.java
@@ -27,10 +27,13 @@
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
public class SubscriptionPollRequest {
- private static final Logger LOGGER = LoggerFactory.getLogger(SubscriptionPollResponse.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(SubscriptionPollRequest.class);
private final transient short requestType;
@@ -41,15 +44,31 @@ public class SubscriptionPollRequest {
/** The maximum size, in bytes, for the response payload. */
private final transient long maxBytes;
+ /**
+ * Per-topic writer-based progress used by the new consensus subscription model. This preserves
+ * topic boundaries while allowing the consumer to provide a recovery hint on reconnect.
+ */
+ private final transient Map progressByTopic;
+
public SubscriptionPollRequest(
final short requestType,
final SubscriptionPollPayload payload,
final long timeoutMs,
final long maxBytes) {
+ this(requestType, payload, timeoutMs, maxBytes, Collections.emptyMap());
+ }
+
+ public SubscriptionPollRequest(
+ final short requestType,
+ final SubscriptionPollPayload payload,
+ final long timeoutMs,
+ final long maxBytes,
+ final Map progressByTopic) {
this.requestType = requestType;
this.payload = payload;
this.timeoutMs = timeoutMs;
this.maxBytes = maxBytes;
+ this.progressByTopic = progressByTopic != null ? progressByTopic : Collections.emptyMap();
}
public short getRequestType() {
@@ -68,6 +87,10 @@ public long getMaxBytes() {
return maxBytes;
}
+ public Map getProgressByTopic() {
+ return progressByTopic;
+ }
+
//////////////////////////// serialization ////////////////////////////
public static ByteBuffer serialize(final SubscriptionPollRequest request) throws IOException {
@@ -83,6 +106,11 @@ private void serialize(final DataOutputStream stream) throws IOException {
payload.serialize(stream);
ReadWriteIOUtils.write(timeoutMs, stream);
ReadWriteIOUtils.write(maxBytes, stream);
+ ReadWriteIOUtils.write(progressByTopic.size(), stream);
+ for (final Map.Entry entry : progressByTopic.entrySet()) {
+ ReadWriteIOUtils.write(entry.getKey(), stream);
+ entry.getValue().serialize(stream);
+ }
}
public static SubscriptionPollRequest deserialize(final ByteBuffer buffer) {
@@ -109,7 +137,20 @@ public static SubscriptionPollRequest deserialize(final ByteBuffer buffer) {
final long timeoutMs = ReadWriteIOUtils.readLong(buffer);
final long maxBytes = ReadWriteIOUtils.readLong(buffer);
- return new SubscriptionPollRequest(requestType, payload, timeoutMs, maxBytes);
+
+ Map progressByTopic = Collections.emptyMap();
+ if (buffer.hasRemaining()) {
+ final int mapSize = ReadWriteIOUtils.readInt(buffer);
+ if (mapSize > 0) {
+ progressByTopic = new HashMap<>(mapSize);
+ for (int i = 0; i < mapSize; i++) {
+ progressByTopic.put(
+ ReadWriteIOUtils.readString(buffer), TopicProgress.deserialize(buffer));
+ }
+ }
+ }
+
+ return new SubscriptionPollRequest(requestType, payload, timeoutMs, maxBytes, progressByTopic);
}
/////////////////////////////// object ///////////////////////////////
@@ -117,13 +158,15 @@ public static SubscriptionPollRequest deserialize(final ByteBuffer buffer) {
@Override
public String toString() {
return "SubscriptionPollRequest{requestType="
- + SubscriptionPollRequestType.valueOf(requestType).toString()
+ + SubscriptionPollRequestType.valueOf(requestType)
+ ", payload="
+ payload
+ ", timeoutMs="
+ timeoutMs
+ ", maxBytes="
+ maxBytes
+ + ", progressByTopic.size="
+ + progressByTopic.size()
+ "}";
}
}
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponse.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponse.java
index 06baa30acee9f..df1bb91a9f3e9 100644
--- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponse.java
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponse.java
@@ -100,6 +100,9 @@ public static SubscriptionPollResponse deserialize(final ByteBuffer buffer) {
case TERMINATION:
payload = new TerminationPayload().deserialize(buffer);
break;
+ case WATERMARK:
+ payload = new WatermarkPayload().deserialize(buffer);
+ break;
default:
LOGGER.warn("unexpected response type: {}, payload will be null", responseType);
break;
@@ -121,9 +124,10 @@ public String toString() {
protected Map coreReportMessage() {
final Map result = new HashMap<>();
- result.put("responseType", SubscriptionPollResponseType.valueOf(responseType).toString());
- result.put("payload", payload.toString());
- result.put("commitContext", commitContext.toString());
+ final SubscriptionPollResponseType type = SubscriptionPollResponseType.valueOf(responseType);
+ result.put("responseType", type != null ? type.toString() : "UNKNOWN(" + responseType + ")");
+ result.put("payload", payload != null ? payload.toString() : "null");
+ result.put("commitContext", commitContext != null ? commitContext.toString() : "null");
return result;
}
}
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponseType.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponseType.java
index b27791b36c538..4ca6cb09dd67c 100644
--- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponseType.java
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponseType.java
@@ -33,6 +33,13 @@ public enum SubscriptionPollResponseType {
FILE_SEAL((short) 4),
TERMINATION((short) 5),
+
+ /**
+ * Periodic timestamp-progress signal from the server-side {@code ConsensusPrefetchingQueue}.
+ * Carries the maximum data timestamp observed so far for a region, enabling client-side watermark
+ * computation even when a region is idle (no new data).
+ */
+ WATERMARK((short) 7),
;
private final short type;
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/TopicProgress.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/TopicProgress.java
new file mode 100644
index 0000000000000..35dfd2e0ca33d
--- /dev/null
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/TopicProgress.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.rpc.subscription.payload.poll;
+
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Objects;
+
+public class TopicProgress {
+
+ private final Map regionProgress;
+
+ public TopicProgress(final Map regionProgress) {
+ this.regionProgress =
+ regionProgress == null
+ ? Collections.emptyMap()
+ : Collections.unmodifiableMap(new LinkedHashMap<>(regionProgress));
+ }
+
+ public Map getRegionProgress() {
+ return regionProgress;
+ }
+
+ public static ByteBuffer serialize(final TopicProgress progress) throws IOException {
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ progress.serialize(outputStream);
+ return ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+ }
+
+ public void serialize(final DataOutputStream stream) throws IOException {
+ ReadWriteIOUtils.write(regionProgress.size(), stream);
+ for (final Map.Entry entry : regionProgress.entrySet()) {
+ ReadWriteIOUtils.write(entry.getKey(), stream);
+ entry.getValue().serialize(stream);
+ }
+ }
+
+ public static TopicProgress deserialize(final ByteBuffer buffer) {
+ final int size = ReadWriteIOUtils.readInt(buffer);
+ final Map regionProgress = new LinkedHashMap<>(size);
+ for (int i = 0; i < size; i++) {
+ regionProgress.put(ReadWriteIOUtils.readString(buffer), RegionProgress.deserialize(buffer));
+ }
+ return new TopicProgress(regionProgress);
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof TopicProgress)) {
+ return false;
+ }
+ final TopicProgress that = (TopicProgress) obj;
+ return Objects.equals(regionProgress, that.regionProgress);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(regionProgress);
+ }
+
+ @Override
+ public String toString() {
+ return "TopicProgress{" + "regionProgress=" + regionProgress + '}';
+ }
+}
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/WatermarkPayload.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/WatermarkPayload.java
new file mode 100644
index 0000000000000..32dab88967497
--- /dev/null
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/WatermarkPayload.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.rpc.subscription.payload.poll;
+
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * Payload for {@link SubscriptionPollResponseType#WATERMARK}.
+ *
+ * Periodically injected by the server-side {@code ConsensusPrefetchingQueue} to report timestamp
+ * progress for a region. Carries the maximum data timestamp observed so far, enabling client-side
+ * {@code WatermarkProcessor} to advance its watermark even when a region is idle (no new data).
+ *
+ *
The {@code dataNodeId} identifies which DataNode emitted this watermark, allowing the client
+ * to track per-node progress across leader transitions.
+ */
+public class WatermarkPayload implements SubscriptionPollPayload {
+
+ /** Maximum data timestamp observed across all InsertNodes in this region's queue. */
+ private transient long watermarkTimestamp;
+
+ /** The DataNode ID that emitted this watermark. */
+ private transient int dataNodeId;
+
+ public WatermarkPayload() {}
+
+ public WatermarkPayload(final long watermarkTimestamp, final int dataNodeId) {
+ this.watermarkTimestamp = watermarkTimestamp;
+ this.dataNodeId = dataNodeId;
+ }
+
+ public long getWatermarkTimestamp() {
+ return watermarkTimestamp;
+ }
+
+ public int getDataNodeId() {
+ return dataNodeId;
+ }
+
+ @Override
+ public void serialize(final DataOutputStream stream) throws IOException {
+ ReadWriteIOUtils.write(watermarkTimestamp, stream);
+ ReadWriteIOUtils.write(dataNodeId, stream);
+ }
+
+ @Override
+ public SubscriptionPollPayload deserialize(final ByteBuffer buffer) {
+ watermarkTimestamp = ReadWriteIOUtils.readLong(buffer);
+ dataNodeId = ReadWriteIOUtils.readInt(buffer);
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ return "WatermarkPayload{watermarkTimestamp="
+ + watermarkTimestamp
+ + ", dataNodeId="
+ + dataNodeId
+ + '}';
+ }
+}
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/WriterId.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/WriterId.java
new file mode 100644
index 0000000000000..ce21e07fe008d
--- /dev/null
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/WriterId.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.rpc.subscription.payload.poll;
+
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+public class WriterId {
+
+ private final String regionId;
+ private final int nodeId;
+ private final long writerEpoch;
+
+ public WriterId(final String regionId, final int nodeId, final long writerEpoch) {
+ this.regionId = regionId;
+ this.nodeId = nodeId;
+ this.writerEpoch = writerEpoch;
+ }
+
+ public String getRegionId() {
+ return regionId;
+ }
+
+ public int getNodeId() {
+ return nodeId;
+ }
+
+ public long getWriterEpoch() {
+ return writerEpoch;
+ }
+
+ public void serialize(final DataOutputStream stream) throws IOException {
+ ReadWriteIOUtils.write(regionId, stream);
+ ReadWriteIOUtils.write(nodeId, stream);
+ ReadWriteIOUtils.write(writerEpoch, stream);
+ }
+
+ public static WriterId deserialize(final ByteBuffer buffer) {
+ return new WriterId(
+ ReadWriteIOUtils.readString(buffer),
+ ReadWriteIOUtils.readInt(buffer),
+ ReadWriteIOUtils.readLong(buffer));
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof WriterId)) {
+ return false;
+ }
+ final WriterId that = (WriterId) obj;
+ return nodeId == that.nodeId
+ && writerEpoch == that.writerEpoch
+ && Objects.equals(regionId, that.regionId);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(regionId, nodeId, writerEpoch);
+ }
+
+ @Override
+ public String toString() {
+ return "WriterId{"
+ + "regionId='"
+ + regionId
+ + '\''
+ + ", nodeId="
+ + nodeId
+ + ", writerEpoch="
+ + writerEpoch
+ + '}';
+ }
+}
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/WriterProgress.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/WriterProgress.java
new file mode 100644
index 0000000000000..f38ea770e8ff6
--- /dev/null
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/WriterProgress.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.rpc.subscription.payload.poll;
+
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+public class WriterProgress {
+
+ private final long physicalTime;
+ private final long localSeq;
+
+ public WriterProgress(final long physicalTime, final long localSeq) {
+ this.physicalTime = physicalTime;
+ this.localSeq = localSeq;
+ }
+
+ public long getPhysicalTime() {
+ return physicalTime;
+ }
+
+ public long getLocalSeq() {
+ return localSeq;
+ }
+
+ public void serialize(final DataOutputStream stream) throws IOException {
+ ReadWriteIOUtils.write(physicalTime, stream);
+ ReadWriteIOUtils.write(localSeq, stream);
+ }
+
+ public static WriterProgress deserialize(final ByteBuffer buffer) {
+ return new WriterProgress(ReadWriteIOUtils.readLong(buffer), ReadWriteIOUtils.readLong(buffer));
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof WriterProgress)) {
+ return false;
+ }
+ final WriterProgress that = (WriterProgress) obj;
+ return physicalTime == that.physicalTime && localSeq == that.localSeq;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(physicalTime, localSeq);
+ }
+
+ @Override
+ public String toString() {
+ return "WriterProgress{" + "physicalTime=" + physicalTime + ", localSeq=" + localSeq + '}';
+ }
+}
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/request/PipeSubscribeRequestType.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/request/PipeSubscribeRequestType.java
index d649aa567ade4..9fcc1d86b0c75 100644
--- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/request/PipeSubscribeRequestType.java
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/request/PipeSubscribeRequestType.java
@@ -31,6 +31,7 @@ public enum PipeSubscribeRequestType {
CLOSE((short) 4),
SUBSCRIBE((short) 5),
UNSUBSCRIBE((short) 6),
+ SEEK((short) 7),
;
private final short type;
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/request/PipeSubscribeSeekReq.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/request/PipeSubscribeSeekReq.java
new file mode 100644
index 0000000000000..e2a78227a6dc5
--- /dev/null
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/request/PipeSubscribeSeekReq.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.rpc.subscription.payload.request;
+
+import org.apache.iotdb.rpc.subscription.payload.poll.TopicProgress;
+import org.apache.iotdb.service.rpc.thrift.TPipeSubscribeReq;
+
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.Objects;
+
+public class PipeSubscribeSeekReq extends TPipeSubscribeReq {
+
+ /** Seek type constants. */
+ public static final short SEEK_TO_BEGINNING = 1;
+
+ public static final short SEEK_TO_END = 2;
+ public static final short SEEK_TO_TIMESTAMP = 3;
+ public static final short SEEK_TO_TOPIC_PROGRESS = 6;
+ public static final short SEEK_AFTER_TOPIC_PROGRESS = 7;
+
+ private transient String topicName;
+ private transient short seekType;
+ private transient long timestamp; // only meaningful when seekType == SEEK_TO_TIMESTAMP
+ private transient TopicProgress topicProgress = new TopicProgress(Collections.emptyMap());
+
+ public String getTopicName() {
+ return topicName;
+ }
+
+ public short getSeekType() {
+ return seekType;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public TopicProgress getTopicProgress() {
+ return topicProgress;
+ }
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ /**
+ * Serialize the incoming parameters into {@code PipeSubscribeSeekReq}, called by the subscription
+ * client.
+ */
+ public static PipeSubscribeSeekReq toTPipeSubscribeReq(
+ final String topicName, final short seekType, final long timestamp) throws IOException {
+ return toTPipeSubscribeReq(topicName, seekType, timestamp, null);
+ }
+
+ public static PipeSubscribeSeekReq toTPipeSubscribeReq(
+ final String topicName, final TopicProgress topicProgress) throws IOException {
+ return toTPipeSubscribeReq(topicName, SEEK_TO_TOPIC_PROGRESS, 0, topicProgress);
+ }
+
+ public static PipeSubscribeSeekReq toTPipeSubscribeSeekAfterReq(
+ final String topicName, final TopicProgress topicProgress) throws IOException {
+ return toTPipeSubscribeReq(topicName, SEEK_AFTER_TOPIC_PROGRESS, 0, topicProgress);
+ }
+
+ public static PipeSubscribeSeekReq toTPipeSubscribeReq(
+ final String topicName,
+ final short seekType,
+ final long timestamp,
+ final TopicProgress topicProgress)
+ throws IOException {
+ final PipeSubscribeSeekReq req = new PipeSubscribeSeekReq();
+
+ req.topicName = topicName;
+ req.seekType = seekType;
+ req.timestamp = timestamp;
+ req.topicProgress =
+ Objects.nonNull(topicProgress) ? topicProgress : new TopicProgress(Collections.emptyMap());
+
+ req.version = PipeSubscribeRequestVersion.VERSION_1.getVersion();
+ req.type = PipeSubscribeRequestType.SEEK.getType();
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(topicName, outputStream);
+ ReadWriteIOUtils.write(seekType, outputStream);
+ if (seekType == SEEK_TO_TIMESTAMP) {
+ ReadWriteIOUtils.write(timestamp, outputStream);
+ } else if (seekType == SEEK_TO_TOPIC_PROGRESS || seekType == SEEK_AFTER_TOPIC_PROGRESS) {
+ req.topicProgress.serialize(outputStream);
+ }
+ req.body = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+
+ return req;
+ }
+
+ /**
+ * Deserialize {@code TPipeSubscribeReq} to obtain parameters, called by the subscription server.
+ */
+ public static PipeSubscribeSeekReq fromTPipeSubscribeReq(final TPipeSubscribeReq seekReq) {
+ final PipeSubscribeSeekReq req = new PipeSubscribeSeekReq();
+
+ if (Objects.nonNull(seekReq.body) && seekReq.body.hasRemaining()) {
+ req.topicName = ReadWriteIOUtils.readString(seekReq.body);
+ req.seekType = ReadWriteIOUtils.readShort(seekReq.body);
+ if (req.seekType == SEEK_TO_TIMESTAMP) {
+ req.timestamp = ReadWriteIOUtils.readLong(seekReq.body);
+ } else if (req.seekType == SEEK_TO_TOPIC_PROGRESS
+ || req.seekType == SEEK_AFTER_TOPIC_PROGRESS) {
+ req.topicProgress = TopicProgress.deserialize(seekReq.body);
+ }
+ }
+
+ req.version = seekReq.version;
+ req.type = seekReq.type;
+ req.body = seekReq.body;
+
+ return req;
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ final PipeSubscribeSeekReq that = (PipeSubscribeSeekReq) obj;
+ return Objects.equals(this.topicName, that.topicName)
+ && this.seekType == that.seekType
+ && this.timestamp == that.timestamp
+ && Objects.equals(this.topicProgress, that.topicProgress)
+ && this.version == that.version
+ && this.type == that.type
+ && Objects.equals(this.body, that.body);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(topicName, seekType, timestamp, topicProgress, version, type, body);
+ }
+}
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeSeekResp.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeSeekResp.java
new file mode 100644
index 0000000000000..c6ea90d5bb069
--- /dev/null
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeSeekResp.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.rpc.subscription.payload.response;
+
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.service.rpc.thrift.TPipeSubscribeResp;
+
+import java.util.Objects;
+
+public class PipeSubscribeSeekResp extends TPipeSubscribeResp {
+
+ /////////////////////////////// Thrift ///////////////////////////////
+
+ /**
+ * Serialize the incoming parameters into {@code PipeSubscribeSeekResp}, called by the
+ * subscription server.
+ */
+ public static PipeSubscribeSeekResp toTPipeSubscribeResp(final TSStatus status) {
+ final PipeSubscribeSeekResp resp = new PipeSubscribeSeekResp();
+
+ resp.status = status;
+ resp.version = PipeSubscribeResponseVersion.VERSION_1.getVersion();
+ resp.type = PipeSubscribeResponseType.ACK.getType();
+
+ return resp;
+ }
+
+ /**
+ * Deserialize {@code TPipeSubscribeResp} to obtain parameters, called by the subscription client.
+ */
+ public static PipeSubscribeSeekResp fromTPipeSubscribeResp(final TPipeSubscribeResp seekResp) {
+ final PipeSubscribeSeekResp resp = new PipeSubscribeSeekResp();
+
+ resp.status = seekResp.status;
+ resp.version = seekResp.version;
+ resp.type = seekResp.type;
+ resp.body = seekResp.body;
+
+ return resp;
+ }
+
+ /////////////////////////////// Object ///////////////////////////////
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ final PipeSubscribeSeekResp that = (PipeSubscribeSeekResp) obj;
+ return Objects.equals(this.status, that.status)
+ && this.version == that.version
+ && this.type == that.type
+ && Objects.equals(this.body, that.body);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(status, version, type, body);
+ }
+}
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/ISubscriptionTablePullConsumer.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/ISubscriptionTablePullConsumer.java
index 0168a1ba3846d..abc5e2de2ff92 100644
--- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/ISubscriptionTablePullConsumer.java
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/ISubscriptionTablePullConsumer.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.session.subscription.consumer;
import org.apache.iotdb.rpc.subscription.exception.SubscriptionException;
+import org.apache.iotdb.rpc.subscription.payload.poll.TopicProgress;
import org.apache.iotdb.session.subscription.payload.SubscriptionMessage;
import java.time.Duration;
@@ -179,6 +180,19 @@ List poll(final Set topicNames, final Duration time
void commitAsync(
final Iterable messages, final AsyncCommitCallback callback);
+ void seekToBeginning(final String topicName) throws SubscriptionException;
+
+ void seekToEnd(final String topicName) throws SubscriptionException;
+
+ TopicProgress positions(final String topicName) throws SubscriptionException;
+
+ TopicProgress committedPositions(final String topicName) throws SubscriptionException;
+
+ void seek(final String topicName, final TopicProgress topicProgress) throws SubscriptionException;
+
+ void seekAfter(final String topicName, final TopicProgress topicProgress)
+ throws SubscriptionException;
+
/**
* Retrieves the unique identifier of this consumer. If no consumer ID was provided at the time of
* consumer construction, a random globally unique ID is automatically assigned after the consumer
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/ISubscriptionTreePullConsumer.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/ISubscriptionTreePullConsumer.java
index 803b7c51224a4..fc9d55bfe218a 100644
--- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/ISubscriptionTreePullConsumer.java
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/ISubscriptionTreePullConsumer.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.session.subscription.consumer;
import org.apache.iotdb.rpc.subscription.exception.SubscriptionException;
+import org.apache.iotdb.rpc.subscription.payload.poll.TopicProgress;
import org.apache.iotdb.session.subscription.payload.SubscriptionMessage;
import java.time.Duration;
@@ -179,6 +180,19 @@ List poll(final Set topicNames, final Duration time
void commitAsync(
final Iterable messages, final AsyncCommitCallback callback);
+ void seekToBeginning(final String topicName) throws SubscriptionException;
+
+ void seekToEnd(final String topicName) throws SubscriptionException;
+
+ TopicProgress positions(final String topicName) throws SubscriptionException;
+
+ TopicProgress committedPositions(final String topicName) throws SubscriptionException;
+
+ void seek(final String topicName, final TopicProgress topicProgress) throws SubscriptionException;
+
+ void seekAfter(final String topicName, final TopicProgress topicProgress)
+ throws SubscriptionException;
+
/**
* Retrieves the unique identifier of this consumer. If no consumer ID was provided at the time of
* consumer construction, a random globally unique ID is automatically assigned after the consumer
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java
index c290af2b67b98..3008704f26a5d 100644
--- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java
@@ -34,11 +34,17 @@
import org.apache.iotdb.rpc.subscription.payload.poll.FileInitPayload;
import org.apache.iotdb.rpc.subscription.payload.poll.FilePiecePayload;
import org.apache.iotdb.rpc.subscription.payload.poll.FileSealPayload;
+import org.apache.iotdb.rpc.subscription.payload.poll.RegionProgress;
import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext;
import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollPayload;
import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponse;
import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponseType;
import org.apache.iotdb.rpc.subscription.payload.poll.TabletsPayload;
+import org.apache.iotdb.rpc.subscription.payload.poll.TopicProgress;
+import org.apache.iotdb.rpc.subscription.payload.poll.WatermarkPayload;
+import org.apache.iotdb.rpc.subscription.payload.poll.WriterId;
+import org.apache.iotdb.rpc.subscription.payload.poll.WriterProgress;
+import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeSeekReq;
import org.apache.iotdb.session.subscription.consumer.AsyncCommitCallback;
import org.apache.iotdb.session.subscription.payload.SubscriptionMessage;
import org.apache.iotdb.session.subscription.payload.SubscriptionMessageType;
@@ -77,6 +83,7 @@
import java.util.concurrent.Callable;
import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledFuture;
@@ -88,6 +95,7 @@
import static org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponseType.FILE_INIT;
import static org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponseType.TABLETS;
import static org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponseType.TERMINATION;
+import static org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponseType.WATERMARK;
import static org.apache.iotdb.session.subscription.util.SetPartitioner.partition;
abstract class AbstractSubscriptionConsumer implements AutoCloseable {
@@ -121,6 +129,26 @@ abstract class AbstractSubscriptionConsumer implements AutoCloseable {
private final int connectionTimeoutInMs;
private final int maxPollParallelism;
+ /**
+ * The latest watermark timestamp received from the server. Updated when WATERMARK events are
+ * processed and stripped. Consumer users can query this to check timestamp progress.
+ */
+ protected volatile long latestWatermarkTimestamp = Long.MIN_VALUE;
+
+ /** Per-topic current positions used as the consumer-guided positioning hint in poll requests. */
+ private final Map currentPositionsByTopic = new ConcurrentHashMap<>();
+
+ /** Per-topic committed positions used as durable recovery points for explicit seek/checkpoint. */
+ private final Map committedPositionsByTopic = new ConcurrentHashMap<>();
+
+ /**
+ * Ack contexts for consensus messages that were already processed locally but could not be
+ * committed because the original provider became unavailable. They are flushed after the same
+ * topic+region is observed again from a live provider.
+ */
+ private final Map> pendingRedirectAcksByTopicRegion =
+ new ConcurrentHashMap<>();
+
@SuppressWarnings("java:S3077")
protected volatile Map subscribedTopics = new HashMap<>();
@@ -376,6 +404,106 @@ private void unsubscribe(Set topicNames, final boolean needParse)
providers.acquireReadLock();
try {
unsubscribeWithRedirection(topicNames);
+ topicNames.forEach(this::clearPendingRedirectAcks);
+ } finally {
+ providers.releaseReadLock();
+ }
+ }
+
+ /////////////////////////////// seek ///////////////////////////////
+
+ /**
+ * Seeks to the earliest available WAL position. Actual position depends on WAL retention — old
+ * segments may have been reclaimed.
+ */
+ public void seekToBeginning(final String topicName) throws SubscriptionException {
+ checkIfOpened();
+ seekInternal(topicName, PipeSubscribeSeekReq.SEEK_TO_BEGINNING, 0);
+ clearCurrentPositions(topicName);
+ clearCommittedPositions(topicName);
+ clearPendingRedirectAcks(topicName);
+ }
+
+ /** Seeks to the current WAL tail. Only newly written data will be consumed after this. */
+ public void seekToEnd(final String topicName) throws SubscriptionException {
+ checkIfOpened();
+ seekInternal(topicName, PipeSubscribeSeekReq.SEEK_TO_END, 0);
+ clearCurrentPositions(topicName);
+ clearCommittedPositions(topicName);
+ clearPendingRedirectAcks(topicName);
+ }
+
+ /**
+ * Returns the latest observed per-region positions for the given topic. This is the consumer's
+ * current fetch position hint and is sent back to the server on subsequent poll requests.
+ */
+ public TopicProgress positions(final String topicName) throws SubscriptionException {
+ checkIfOpened();
+ final TopicProgress progress = currentPositionsByTopic.get(topicName);
+ return Objects.nonNull(progress)
+ ? new TopicProgress(progress.getRegionProgress())
+ : new TopicProgress(Collections.emptyMap());
+ }
+
+ /**
+ * Returns the latest committed per-region positions for the given topic. This is the recoverable
+ * checkpoint position that should be persisted by callers.
+ */
+ public TopicProgress committedPositions(final String topicName) throws SubscriptionException {
+ checkIfOpened();
+ final TopicProgress progress = committedPositionsByTopic.get(topicName);
+ return Objects.nonNull(progress)
+ ? new TopicProgress(progress.getRegionProgress())
+ : new TopicProgress(Collections.emptyMap());
+ }
+
+ public void seek(final String topicName, final TopicProgress topicProgress)
+ throws SubscriptionException {
+ checkIfOpened();
+ final TopicProgress safeProgress =
+ Objects.nonNull(topicProgress) ? topicProgress : new TopicProgress(Collections.emptyMap());
+ seekInternalTopicProgress(topicName, safeProgress);
+ overlayCurrentPositions(topicName, safeProgress);
+ overlayCommittedPositions(topicName, safeProgress);
+ clearPendingRedirectAcks(topicName);
+ }
+
+ public void seekAfter(final String topicName, final TopicProgress topicProgress)
+ throws SubscriptionException {
+ checkIfOpened();
+ final TopicProgress safeProgress =
+ Objects.nonNull(topicProgress) ? topicProgress : new TopicProgress(Collections.emptyMap());
+ seekAfterInternalTopicProgress(topicName, safeProgress);
+ overlayCurrentPositions(topicName, safeProgress);
+ overlayCommittedPositions(topicName, safeProgress);
+ clearPendingRedirectAcks(topicName);
+ }
+
+ private void seekInternal(final String topicName, final short seekType, final long timestamp)
+ throws SubscriptionException {
+ providers.acquireReadLock();
+ try {
+ seekWithRedirection(topicName, seekType, timestamp);
+ } finally {
+ providers.releaseReadLock();
+ }
+ }
+
+ private void seekInternalTopicProgress(final String topicName, final TopicProgress topicProgress)
+ throws SubscriptionException {
+ providers.acquireReadLock();
+ try {
+ seekWithRedirectionTopicProgress(topicName, topicProgress);
+ } finally {
+ providers.releaseReadLock();
+ }
+ }
+
+ private void seekAfterInternalTopicProgress(
+ final String topicName, final TopicProgress topicProgress) throws SubscriptionException {
+ providers.acquireReadLock();
+ try {
+ seekAfterWithRedirectionTopicProgress(topicName, topicProgress);
} finally {
providers.releaseReadLock();
}
@@ -522,9 +650,44 @@ private Path getFilePath(
unsubscribe(Collections.singleton(topicNameToUnsubscribe), false);
return Optional.empty();
});
+ put(
+ WATERMARK,
+ (resp, timer) -> {
+ final SubscriptionCommitContext commitContext = resp.getCommitContext();
+ final WatermarkPayload payload = (WatermarkPayload) resp.getPayload();
+ return Optional.of(
+ new SubscriptionMessage(
+ commitContext, payload.getWatermarkTimestamp()));
+ });
}
});
+ /**
+ * Returns the set of DataNode IDs for providers that are currently available. Used by subclasses
+ * to detect unavailable DataNodes and notify the epoch ordering processor.
+ */
+ protected Set getAvailableDataNodeIds() {
+ providers.acquireReadLock();
+ try {
+ final Set ids = new HashSet<>();
+ for (final AbstractSubscriptionProvider provider : providers.getAllAvailableProviders()) {
+ ids.add(provider.getDataNodeId());
+ }
+ return ids;
+ } finally {
+ providers.releaseReadLock();
+ }
+ }
+
+ /**
+ * Returns the latest watermark timestamp received from the server. This tracks the maximum data
+ * timestamp observed across all polled regions. Returns {@code Long.MIN_VALUE} if no watermark
+ * has been received yet.
+ */
+ public long getLatestWatermarkTimestamp() {
+ return latestWatermarkTimestamp;
+ }
+
protected List multiplePoll(
/* @NotNull */ final Set topicNames, final long timeoutMs) {
if (topicNames.isEmpty()) {
@@ -685,6 +848,8 @@ private List singlePoll(
// add all current messages to result messages
messages.addAll(currentMessages);
+ advanceCurrentPositions(currentMessages);
+ flushPendingRedirectAcks(currentMessages);
// TODO: maybe we can poll a few more times
if (!messages.isEmpty()) {
@@ -1079,7 +1244,7 @@ private List pollInternal(
}
// ignore SubscriptionConnectionException to improve poll auto retry
try {
- return provider.poll(topicNames, timeoutMs);
+ return provider.poll(topicNames, timeoutMs, buildCurrentProgressByTopic(topicNames));
} catch (final SubscriptionConnectionException ignored) {
return Collections.emptyList();
}
@@ -1155,7 +1320,59 @@ protected void ack(final Iterable messages) throws Subscrip
for (final Entry> entry :
dataNodeIdToSubscriptionCommitContexts.entrySet()) {
commitInternal(entry.getKey(), entry.getValue(), false);
+ advanceCommittedPositions(entry.getValue());
+ }
+ }
+
+ protected Set ackWithPartialProgress(
+ final Iterable messages) throws SubscriptionException {
+ final Map> dataNodeIdToMessages = new HashMap<>();
+ for (final SubscriptionMessage message : messages) {
+ dataNodeIdToMessages
+ .computeIfAbsent(message.getCommitContext().getDataNodeId(), ignored -> new ArrayList<>())
+ .add(message);
+ }
+
+ final Set removableMessages = new HashSet<>();
+ for (final Entry> entry : dataNodeIdToMessages.entrySet()) {
+ final List commitContexts =
+ entry.getValue().stream()
+ .map(SubscriptionMessage::getCommitContext)
+ .collect(Collectors.toList());
+ try {
+ commitInternal(entry.getKey(), commitContexts, false);
+ advanceCommittedPositions(commitContexts);
+ removableMessages.addAll(entry.getValue());
+ } catch (final SubscriptionConnectionException e) {
+ int stagedCount = 0;
+ int retainedCount = 0;
+ for (final SubscriptionMessage message : entry.getValue()) {
+ if (isConsensusCommitContext(message.getCommitContext())) {
+ stagePendingRedirectAck(message.getCommitContext());
+ removableMessages.add(message);
+ stagedCount++;
+ } else {
+ retainedCount++;
+ }
+ }
+ if (stagedCount > 0) {
+ LOGGER.warn(
+ "{} staged {} consensus ack(s) for redirect after provider {} became unavailable",
+ this,
+ stagedCount,
+ entry.getKey());
+ }
+ if (retainedCount > 0) {
+ LOGGER.warn(
+ "{} keep {} non-consensus ack(s) pending after provider {} commit failure",
+ this,
+ retainedCount,
+ entry.getKey(),
+ e);
+ }
+ }
}
+ return removableMessages;
}
protected void nack(final Iterable messages) throws SubscriptionException {
@@ -1383,6 +1600,346 @@ private void unsubscribeWithRedirection(final Set topicNames)
throw new SubscriptionRuntimeCriticalException(errorMessage);
}
+ /**
+ * Sends seek request to ALL available providers. Unlike subscribe/unsubscribe, seek must reach
+ * every node because data regions for the topic may be distributed across different nodes.
+ */
+ private void seekWithRedirection(
+ final String topicName, final short seekType, final long timestamp)
+ throws SubscriptionException {
+ final List providers = this.providers.getAllAvailableProviders();
+ if (providers.isEmpty()) {
+ throw new SubscriptionConnectionException(
+ String.format(
+ "Cluster has no available subscription providers when %s seek topic %s",
+ this, topicName));
+ }
+ boolean anySuccess = false;
+ for (final AbstractSubscriptionProvider provider : providers) {
+ try {
+ provider.seek(topicName, seekType, timestamp);
+ anySuccess = true;
+ } catch (final Exception e) {
+ LOGGER.warn(
+ "{} failed to seek topic {} from subscription provider {}, continuing with other providers...",
+ this,
+ topicName,
+ provider,
+ e);
+ }
+ }
+ if (!anySuccess) {
+ final String errorMessage =
+ String.format(
+ "%s failed to seek topic %s from all available subscription providers %s",
+ this, topicName, providers);
+ LOGGER.warn(errorMessage);
+ throw new SubscriptionRuntimeCriticalException(errorMessage);
+ }
+ }
+
+ private void seekWithRedirectionTopicProgress(
+ final String topicName, final TopicProgress topicProgress) throws SubscriptionException {
+ final List providers = this.providers.getAllAvailableProviders();
+ if (providers.isEmpty()) {
+ throw new SubscriptionConnectionException(
+ String.format(
+ "Cluster has no available subscription providers when %s seek topic %s",
+ this, topicName));
+ }
+ boolean anySuccess = false;
+ for (final AbstractSubscriptionProvider provider : providers) {
+ try {
+ provider.seekToTopicProgress(topicName, topicProgress);
+ anySuccess = true;
+ } catch (final Exception e) {
+ LOGGER.warn(
+ "{} failed to seek topic {} to topicProgress(regionCount={}) from provider {}, continuing...",
+ this,
+ topicName,
+ topicProgress.getRegionProgress().size(),
+ provider,
+ e);
+ }
+ }
+ if (!anySuccess) {
+ final String errorMessage =
+ String.format(
+ "%s failed to seek topic %s to topicProgress(regionCount=%d) from all providers %s",
+ this, topicName, topicProgress.getRegionProgress().size(), providers);
+ LOGGER.warn(errorMessage);
+ throw new SubscriptionRuntimeCriticalException(errorMessage);
+ }
+ }
+
+ private void seekAfterWithRedirectionTopicProgress(
+ final String topicName, final TopicProgress topicProgress) throws SubscriptionException {
+ final List providers = this.providers.getAllAvailableProviders();
+ if (providers.isEmpty()) {
+ throw new SubscriptionConnectionException(
+ String.format(
+ "Cluster has no available subscription providers when %s seekAfter topic %s",
+ this, topicName));
+ }
+ boolean anySuccess = false;
+ for (final AbstractSubscriptionProvider provider : providers) {
+ try {
+ provider.seekAfterTopicProgress(topicName, topicProgress);
+ anySuccess = true;
+ } catch (final Exception e) {
+ LOGGER.warn(
+ "{} failed to seekAfter topic {} to topicProgress(regionCount={}) from provider {}, continuing...",
+ this,
+ topicName,
+ topicProgress.getRegionProgress().size(),
+ provider,
+ e);
+ }
+ }
+ if (!anySuccess) {
+ final String errorMessage =
+ String.format(
+ "%s failed to seekAfter topic %s to topicProgress(regionCount=%d) from all providers %s",
+ this, topicName, topicProgress.getRegionProgress().size(), providers);
+ LOGGER.warn(errorMessage);
+ throw new SubscriptionRuntimeCriticalException(errorMessage);
+ }
+ }
+
+ private Map buildCurrentProgressByTopic(final Set topicNames) {
+ final Map result = new HashMap<>();
+ for (final String topicName : topicNames) {
+ final TopicProgress topicProgress = currentPositionsByTopic.get(topicName);
+ if (Objects.isNull(topicProgress) || topicProgress.getRegionProgress().isEmpty()) {
+ continue;
+ }
+ result.put(topicName, new TopicProgress(topicProgress.getRegionProgress()));
+ }
+ return result;
+ }
+
+ private void advanceCurrentPositions(final List messages) {
+ for (final SubscriptionMessage message : messages) {
+ final SubscriptionCommitContext commitContext = message.getCommitContext();
+ if (Objects.isNull(commitContext) || Objects.isNull(commitContext.getTopicName())) {
+ continue;
+ }
+ mergeTopicProgress(
+ currentPositionsByTopic,
+ commitContext.getTopicName(),
+ extractWriterId(commitContext),
+ extractWriterProgress(commitContext));
+ }
+ }
+
+ private void advanceCommittedPositions(
+ final List subscriptionCommitContexts) {
+ for (final SubscriptionCommitContext commitContext : subscriptionCommitContexts) {
+ if (Objects.isNull(commitContext) || Objects.isNull(commitContext.getTopicName())) {
+ continue;
+ }
+ mergeTopicProgress(
+ committedPositionsByTopic,
+ commitContext.getTopicName(),
+ extractWriterId(commitContext),
+ extractWriterProgress(commitContext));
+ }
+ }
+
+ private boolean isConsensusCommitContext(final SubscriptionCommitContext commitContext) {
+ return Objects.nonNull(commitContext)
+ && Objects.nonNull(commitContext.getWriterId())
+ && Objects.nonNull(commitContext.getWriterProgress())
+ && Objects.nonNull(commitContext.getRegionId())
+ && !commitContext.getRegionId().isEmpty();
+ }
+
+ private String buildTopicRegionKey(final SubscriptionCommitContext commitContext) {
+ return commitContext.getTopicName() + '\u0001' + commitContext.getRegionId();
+ }
+
+ private void stagePendingRedirectAck(final SubscriptionCommitContext commitContext) {
+ pendingRedirectAcksByTopicRegion
+ .computeIfAbsent(
+ buildTopicRegionKey(commitContext), ignored -> ConcurrentHashMap.newKeySet())
+ .add(commitContext);
+ }
+
+ private void flushPendingRedirectAcks(final List currentMessages) {
+ final Map redirectTargetByTopicRegion = new HashMap<>();
+ for (final SubscriptionMessage message : currentMessages) {
+ final SubscriptionCommitContext commitContext = message.getCommitContext();
+ if (!isConsensusCommitContext(commitContext)) {
+ continue;
+ }
+ redirectTargetByTopicRegion.put(
+ buildTopicRegionKey(commitContext), commitContext.getDataNodeId());
+ }
+
+ for (final Entry entry : redirectTargetByTopicRegion.entrySet()) {
+ final Set pendingContexts =
+ pendingRedirectAcksByTopicRegion.get(entry.getKey());
+ if (Objects.isNull(pendingContexts) || pendingContexts.isEmpty()) {
+ continue;
+ }
+
+ final List contextsToRedirect = new ArrayList<>(pendingContexts);
+ try {
+ commitInternal(entry.getValue(), contextsToRedirect, false);
+ advanceCommittedPositions(contextsToRedirect);
+ contextsToRedirect.forEach(pendingContexts::remove);
+ if (pendingContexts.isEmpty()) {
+ pendingRedirectAcksByTopicRegion.remove(entry.getKey(), pendingContexts);
+ }
+ } catch (final SubscriptionException e) {
+ LOGGER.warn(
+ "{} failed to redirect {} pending consensus ack(s) for {} via provider {}",
+ this,
+ contextsToRedirect.size(),
+ entry.getKey(),
+ entry.getValue(),
+ e);
+ }
+ }
+ }
+
+ private boolean isNewerPosition(
+ final long newEpoch, final long newSyncIndex, final long oldEpoch, final long oldSyncIndex) {
+ return newEpoch > oldEpoch || (newEpoch == oldEpoch && newSyncIndex > oldSyncIndex);
+ }
+
+ private void clearCurrentPositions(final String topicName) {
+ currentPositionsByTopic.remove(topicName);
+ }
+
+ private void clearCommittedPositions(final String topicName) {
+ committedPositionsByTopic.remove(topicName);
+ }
+
+ private void clearPendingRedirectAcks(final String topicName) {
+ final String prefix = topicName + '\u0001';
+ pendingRedirectAcksByTopicRegion.keySet().removeIf(key -> key.startsWith(prefix));
+ }
+
+ private void setCurrentPositions(final String topicName, final TopicProgress topicProgress) {
+ if (Objects.isNull(topicProgress) || topicProgress.getRegionProgress().isEmpty()) {
+ currentPositionsByTopic.remove(topicName);
+ return;
+ }
+ currentPositionsByTopic.put(topicName, new TopicProgress(topicProgress.getRegionProgress()));
+ }
+
+ private void setCommittedPositions(final String topicName, final TopicProgress topicProgress) {
+ if (Objects.isNull(topicProgress) || topicProgress.getRegionProgress().isEmpty()) {
+ committedPositionsByTopic.remove(topicName);
+ return;
+ }
+ committedPositionsByTopic.put(topicName, new TopicProgress(topicProgress.getRegionProgress()));
+ }
+
+ private void overlayCurrentPositions(final String topicName, final TopicProgress topicProgress) {
+ overlayTopicProgress(currentPositionsByTopic, topicName, topicProgress);
+ }
+
+ private void overlayCommittedPositions(
+ final String topicName, final TopicProgress topicProgress) {
+ overlayTopicProgress(committedPositionsByTopic, topicName, topicProgress);
+ }
+
+ private void overlayTopicProgress(
+ final Map progressByTopic,
+ final String topicName,
+ final TopicProgress topicProgress) {
+ if (Objects.isNull(topicName)
+ || topicName.isEmpty()
+ || Objects.isNull(topicProgress)
+ || topicProgress.getRegionProgress().isEmpty()) {
+ return;
+ }
+ progressByTopic.compute(
+ topicName,
+ (ignored, oldTopicProgress) -> {
+ final Map mergedRegionProgress =
+ Objects.nonNull(oldTopicProgress)
+ ? new HashMap<>(oldTopicProgress.getRegionProgress())
+ : new HashMap<>();
+ topicProgress
+ .getRegionProgress()
+ .forEach(
+ (regionId, regionProgress) -> {
+ if (Objects.isNull(regionId)
+ || regionId.isEmpty()
+ || Objects.isNull(regionProgress)
+ || regionProgress.getWriterPositions().isEmpty()) {
+ return;
+ }
+ mergedRegionProgress.put(
+ regionId,
+ new RegionProgress(new HashMap<>(regionProgress.getWriterPositions())));
+ });
+ return mergedRegionProgress.isEmpty() ? null : new TopicProgress(mergedRegionProgress);
+ });
+ }
+
+ private WriterId extractWriterId(final SubscriptionCommitContext commitContext) {
+ if (Objects.nonNull(commitContext.getWriterId())) {
+ return commitContext.getWriterId();
+ }
+ if (Objects.isNull(commitContext.getRegionId()) || commitContext.getRegionId().isEmpty()) {
+ return null;
+ }
+ return new WriterId(commitContext.getRegionId(), commitContext.getDataNodeId(), 0L);
+ }
+
+ private WriterProgress extractWriterProgress(final SubscriptionCommitContext commitContext) {
+ if (Objects.nonNull(commitContext.getWriterProgress())) {
+ return commitContext.getWriterProgress();
+ }
+ if (commitContext.getLocalSeq() < 0) {
+ return null;
+ }
+ return new WriterProgress(commitContext.getPhysicalTime(), commitContext.getLocalSeq());
+ }
+
+ private void mergeTopicProgress(
+ final Map progressByTopic,
+ final String topicName,
+ final WriterId writerId,
+ final WriterProgress writerProgress) {
+ if (Objects.isNull(writerId)
+ || Objects.isNull(writerProgress)
+ || Objects.isNull(topicName)
+ || topicName.isEmpty()) {
+ return;
+ }
+ progressByTopic.compute(
+ topicName,
+ (key, oldTopicProgress) -> {
+ final Map regionProgressById =
+ Objects.nonNull(oldTopicProgress)
+ ? new HashMap<>(oldTopicProgress.getRegionProgress())
+ : new HashMap<>();
+ final RegionProgress oldRegionProgress = regionProgressById.get(writerId.getRegionId());
+ final Map writerPositions =
+ Objects.nonNull(oldRegionProgress)
+ ? new HashMap<>(oldRegionProgress.getWriterPositions())
+ : new HashMap<>();
+ writerPositions.merge(
+ writerId,
+ writerProgress,
+ (oldVal, newVal) ->
+ isNewerPosition(
+ newVal.getPhysicalTime(),
+ newVal.getLocalSeq(),
+ oldVal.getPhysicalTime(),
+ oldVal.getLocalSeq())
+ ? newVal
+ : oldVal);
+ regionProgressById.put(writerId.getRegionId(), new RegionProgress(writerPositions));
+ return new TopicProgress(regionProgressById);
+ });
+ }
+
Map fetchAllEndPointsWithRedirection() throws SubscriptionException {
final List providers = this.providers.getAllAvailableProviders();
if (providers.isEmpty()) {
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java
index 7f3582d195d6a..1faadf196a955 100644
--- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java
@@ -37,11 +37,13 @@
import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollRequest;
import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollRequestType;
import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponse;
+import org.apache.iotdb.rpc.subscription.payload.poll.TopicProgress;
import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeCloseReq;
import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeCommitReq;
import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeHandshakeReq;
import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeHeartbeatReq;
import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribePollReq;
+import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeSeekReq;
import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeSubscribeReq;
import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeUnsubscribeReq;
import org.apache.iotdb.rpc.subscription.payload.response.PipeSubscribeHandshakeResp;
@@ -59,6 +61,7 @@
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -332,14 +335,107 @@ Map unsubscribe(final Set topicNames) throws Subscr
return unsubscribeResp.getTopics();
}
+ void seek(final String topicName, final short seekType, final long timestamp)
+ throws SubscriptionException {
+ final PipeSubscribeSeekReq req;
+ try {
+ req = PipeSubscribeSeekReq.toTPipeSubscribeReq(topicName, seekType, timestamp);
+ } catch (final IOException e) {
+ LOGGER.warn(
+ "IOException occurred when SubscriptionProvider {} serialize seek request for topic {}",
+ this,
+ topicName,
+ e);
+ throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e);
+ }
+ final TPipeSubscribeResp resp;
+ try {
+ resp = getSessionConnection().pipeSubscribe(req);
+ } catch (final TException | IoTDBConnectionException e) {
+ LOGGER.warn(
+ "TException/IoTDBConnectionException occurred when SubscriptionProvider {} seek with request for topic {}, set SubscriptionProvider unavailable",
+ this,
+ topicName,
+ e);
+ setUnavailable();
+ throw new SubscriptionConnectionException(e.getMessage(), e);
+ }
+ verifyPipeSubscribeSuccess(resp.status);
+ }
+
+ void seekToTopicProgress(final String topicName, final TopicProgress topicProgress)
+ throws SubscriptionException {
+ final PipeSubscribeSeekReq req;
+ try {
+ req = PipeSubscribeSeekReq.toTPipeSubscribeReq(topicName, topicProgress);
+ } catch (final IOException e) {
+ LOGGER.warn(
+ "IOException occurred when SubscriptionProvider {} serialize seek(topicProgress) for topic {}",
+ this,
+ topicName,
+ e);
+ throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e);
+ }
+ final TPipeSubscribeResp resp;
+ try {
+ resp = getSessionConnection().pipeSubscribe(req);
+ } catch (final TException | IoTDBConnectionException e) {
+ LOGGER.warn(
+ "TException/IoTDBConnectionException occurred when SubscriptionProvider {} seek(topicProgress) for topic {}, set SubscriptionProvider unavailable",
+ this,
+ topicName,
+ e);
+ setUnavailable();
+ throw new SubscriptionConnectionException(e.getMessage(), e);
+ }
+ verifyPipeSubscribeSuccess(resp.status);
+ }
+
+ void seekAfterTopicProgress(final String topicName, final TopicProgress topicProgress)
+ throws SubscriptionException {
+ final PipeSubscribeSeekReq req;
+ try {
+ req = PipeSubscribeSeekReq.toTPipeSubscribeSeekAfterReq(topicName, topicProgress);
+ } catch (final IOException e) {
+ LOGGER.warn(
+ "IOException occurred when SubscriptionProvider {} serialize seekAfter(topicProgress) for topic {}",
+ this,
+ topicName,
+ e);
+ throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e);
+ }
+ final TPipeSubscribeResp resp;
+ try {
+ resp = getSessionConnection().pipeSubscribe(req);
+ } catch (final TException | IoTDBConnectionException e) {
+ LOGGER.warn(
+ "TException/IoTDBConnectionException occurred when SubscriptionProvider {} seekAfter(topicProgress) for topic {}, set SubscriptionProvider unavailable",
+ this,
+ topicName,
+ e);
+ setUnavailable();
+ throw new SubscriptionConnectionException(e.getMessage(), e);
+ }
+ verifyPipeSubscribeSuccess(resp.status);
+ }
+
List poll(final Set topicNames, final long timeoutMs)
throws SubscriptionException {
+ return poll(topicNames, timeoutMs, Collections.emptyMap());
+ }
+
+ List poll(
+ final Set topicNames,
+ final long timeoutMs,
+ final Map progressByTopic)
+ throws SubscriptionException {
return poll(
new SubscriptionPollRequest(
SubscriptionPollRequestType.POLL.getType(),
new PollPayload(topicNames),
timeoutMs,
- session.getThriftMaxFrameSize()));
+ session.getThriftMaxFrameSize(),
+ progressByTopic));
}
List pollFile(
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionPullConsumer.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionPullConsumer.java
index 0c7478fa64dfb..37aed9204b8d8 100644
--- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionPullConsumer.java
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionPullConsumer.java
@@ -21,8 +21,11 @@
import org.apache.iotdb.rpc.subscription.config.ConsumerConstant;
import org.apache.iotdb.rpc.subscription.exception.SubscriptionException;
+import org.apache.iotdb.rpc.subscription.payload.poll.TopicProgress;
import org.apache.iotdb.session.subscription.consumer.AsyncCommitCallback;
+import org.apache.iotdb.session.subscription.payload.PollResult;
import org.apache.iotdb.session.subscription.payload.SubscriptionMessage;
+import org.apache.iotdb.session.subscription.payload.SubscriptionMessageType;
import org.apache.iotdb.session.subscription.util.CollectionUtils;
import org.apache.iotdb.session.subscription.util.IdentifierUtils;
@@ -30,6 +33,7 @@
import org.slf4j.LoggerFactory;
import java.time.Duration;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -64,6 +68,8 @@ public abstract class AbstractSubscriptionPullConsumer extends AbstractSubscript
private final boolean autoCommit;
private final long autoCommitIntervalMs;
+ private final List processors = new ArrayList<>();
+
private SortedMap> uncommittedMessages;
private final AtomicBoolean isClosed = new AtomicBoolean(true);
@@ -134,6 +140,24 @@ public synchronized void close() {
return;
}
+ // flush all processors and commit any remaining buffered messages
+ if (!processors.isEmpty()) {
+ final List flushed = new ArrayList<>();
+ for (final SubscriptionMessageProcessor processor : processors) {
+ final List out = processor.flush();
+ if (out != null) {
+ flushed.addAll(out);
+ }
+ }
+ if (!flushed.isEmpty() && autoCommit) {
+ try {
+ commitSync(flushed);
+ } catch (final SubscriptionException e) {
+ LOGGER.warn("Failed to commit flushed processor messages on close", e);
+ }
+ }
+ }
+
if (autoCommit) {
// commit all uncommitted messages
commitAllUncommittedMessages();
@@ -185,7 +209,7 @@ protected List poll(final Set topicNames, final lon
}
final List messages = multiplePoll(parsedTopicNames, timeoutMs);
- if (messages.isEmpty()) {
+ if (messages.isEmpty() && processors.isEmpty()) {
LOGGER.info(
"SubscriptionPullConsumer {} poll empty message from topics {} after {} millisecond(s)",
this,
@@ -194,6 +218,35 @@ protected List poll(final Set topicNames, final lon
return messages;
}
+ // Apply processor chain if configured
+ List processed = messages;
+ if (!processors.isEmpty()) {
+ for (final SubscriptionMessageProcessor processor : processors) {
+ processed = processor.process(processed);
+ }
+ }
+
+ // Update watermark timestamp before stripping watermark events
+ for (final SubscriptionMessage m : processed) {
+ if (m.getMessageType() == SubscriptionMessageType.WATERMARK.getType()) {
+ final long ts = m.getWatermarkTimestamp();
+ if (ts > latestWatermarkTimestamp) {
+ latestWatermarkTimestamp = ts;
+ }
+ }
+ }
+
+ // Strip system messages — they are only for processors, not for users
+ processed.removeIf(
+ m -> {
+ final short type = m.getMessageType();
+ return type == SubscriptionMessageType.WATERMARK.getType();
+ });
+
+ if (processed.isEmpty()) {
+ return processed;
+ }
+
// add to uncommitted messages
if (autoCommit) {
final long currentTimestamp = System.currentTimeMillis();
@@ -203,10 +256,54 @@ protected List poll(final Set topicNames, final lon
}
uncommittedMessages
.computeIfAbsent(index, o -> new ConcurrentSkipListSet<>())
- .addAll(messages);
+ .addAll(processed);
+ }
+
+ return processed;
+ }
+
+ /////////////////////////////// processor ///////////////////////////////
+
+ /**
+ * Adds a message processor to the pipeline. Processors are applied in order on each poll() call.
+ *
+ * @param processor the processor to add
+ */
+ protected AbstractSubscriptionPullConsumer addProcessor(
+ final SubscriptionMessageProcessor processor) {
+ processors.add(processor);
+ return this;
+ }
+
+ /**
+ * Polls with processor metadata. Returns a {@link PollResult} containing the messages, the total
+ * number of buffered messages across all processors, and the current watermark.
+ */
+ protected PollResult pollWithInfo(final long timeoutMs) throws SubscriptionException {
+ final List messages = poll(timeoutMs);
+ int totalBuffered = 0;
+ long watermark = -1;
+ for (final SubscriptionMessageProcessor processor : processors) {
+ totalBuffered += processor.getBufferedCount();
+ if (processor instanceof WatermarkProcessor) {
+ watermark = ((WatermarkProcessor) processor).getWatermark();
+ }
}
+ return new PollResult(messages, totalBuffered, watermark);
+ }
- return messages;
+ protected PollResult pollWithInfo(final Set topicNames, final long timeoutMs)
+ throws SubscriptionException {
+ final List messages = poll(topicNames, timeoutMs);
+ int totalBuffered = 0;
+ long watermark = -1;
+ for (final SubscriptionMessageProcessor processor : processors) {
+ totalBuffered += processor.getBufferedCount();
+ if (processor instanceof WatermarkProcessor) {
+ watermark = ((WatermarkProcessor) processor).getWatermark();
+ }
+ }
+ return new PollResult(messages, totalBuffered, watermark);
}
/////////////////////////////// commit ///////////////////////////////
@@ -238,6 +335,46 @@ protected void commitAsync(
super.commitAsync(messages, callback);
}
+ /////////////////////////////// seek ///////////////////////////////
+
+ /**
+ * Clears uncommitted auto-commit messages after seek to prevent stale acks from committing events
+ * that belonged to the pre-seek position.
+ */
+ @Override
+ public void seekToBeginning(final String topicName) throws SubscriptionException {
+ super.seekToBeginning(topicName);
+ if (autoCommit) {
+ uncommittedMessages.clear();
+ }
+ }
+
+ @Override
+ public void seekToEnd(final String topicName) throws SubscriptionException {
+ super.seekToEnd(topicName);
+ if (autoCommit) {
+ uncommittedMessages.clear();
+ }
+ }
+
+ @Override
+ public void seek(final String topicName, final TopicProgress topicProgress)
+ throws SubscriptionException {
+ super.seek(topicName, topicProgress);
+ if (autoCommit) {
+ uncommittedMessages.clear();
+ }
+ }
+
+ @Override
+ public void seekAfter(final String topicName, final TopicProgress topicProgress)
+ throws SubscriptionException {
+ super.seekAfter(topicName, topicProgress);
+ if (autoCommit) {
+ uncommittedMessages.clear();
+ }
+ }
+
/////////////////////////////// auto commit ///////////////////////////////
private void submitAutoCommitWorker() {
@@ -274,8 +411,19 @@ public void run() {
for (final Map.Entry> entry :
uncommittedMessages.headMap(index).entrySet()) {
try {
- ack(entry.getValue());
- uncommittedMessages.remove(entry.getKey());
+ final Set removableMessages =
+ ackWithPartialProgress(entry.getValue());
+ if (removableMessages.isEmpty()) {
+ continue;
+ }
+ if (removableMessages.size() == entry.getValue().size()) {
+ uncommittedMessages.remove(entry.getKey());
+ continue;
+ }
+ entry.getValue().removeAll(removableMessages);
+ if (entry.getValue().isEmpty()) {
+ uncommittedMessages.remove(entry.getKey());
+ }
} catch (final Exception e) {
LOGGER.warn("something unexpected happened when auto commit messages...", e);
}
@@ -286,8 +434,18 @@ public void run() {
private void commitAllUncommittedMessages() {
for (final Map.Entry> entry : uncommittedMessages.entrySet()) {
try {
- ack(entry.getValue());
- uncommittedMessages.remove(entry.getKey());
+ final Set removableMessages = ackWithPartialProgress(entry.getValue());
+ if (removableMessages.isEmpty()) {
+ continue;
+ }
+ if (removableMessages.size() == entry.getValue().size()) {
+ uncommittedMessages.remove(entry.getKey());
+ continue;
+ }
+ entry.getValue().removeAll(removableMessages);
+ if (entry.getValue().isEmpty()) {
+ uncommittedMessages.remove(entry.getKey());
+ }
} catch (final Exception e) {
LOGGER.warn("something unexpected happened when commit messages during close", e);
}
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionPushConsumer.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionPushConsumer.java
index 3ff93db218b27..1ac9f08696ddb 100644
--- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionPushConsumer.java
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionPushConsumer.java
@@ -26,6 +26,7 @@
import org.apache.iotdb.session.subscription.consumer.ConsumeResult;
import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePushConsumer;
import org.apache.iotdb.session.subscription.payload.SubscriptionMessage;
+import org.apache.iotdb.session.subscription.payload.SubscriptionMessageType;
import org.apache.iotdb.session.subscription.util.CollectionUtils;
import org.slf4j.Logger;
@@ -180,6 +181,21 @@ public void run() {
try {
final List messages =
multiplePoll(subscribedTopics.keySet(), autoPollTimeoutMs);
+ // Update watermark timestamp before stripping watermark events
+ for (final SubscriptionMessage m : messages) {
+ if (m.getMessageType() == SubscriptionMessageType.WATERMARK.getType()) {
+ final long ts = m.getWatermarkTimestamp();
+ if (ts > latestWatermarkTimestamp) {
+ latestWatermarkTimestamp = ts;
+ }
+ }
+ }
+ // Strip system messages — push consumer does not use processors
+ messages.removeIf(
+ m -> {
+ final short type = m.getMessageType();
+ return type == SubscriptionMessageType.WATERMARK.getType();
+ });
if (messages.isEmpty()) {
LOGGER.info(
"SubscriptionPushConsumer {} poll empty message from topics {} after {} millisecond(s)",
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/ColumnAlignProcessor.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/ColumnAlignProcessor.java
new file mode 100644
index 0000000000000..13910a86c9abe
--- /dev/null
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/ColumnAlignProcessor.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.session.subscription.consumer.base;
+
+import org.apache.iotdb.session.subscription.payload.SubscriptionMessage;
+import org.apache.iotdb.session.subscription.payload.SubscriptionMessageType;
+
+import org.apache.tsfile.utils.BitMap;
+import org.apache.tsfile.write.record.Tablet;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A non-buffering processor that forward-fills null columns in each Tablet using the last known
+ * value for the same device/table. This is useful for CDC scenarios where a write only updates a
+ * subset of columns, leaving others null; the processor fills them with the most recent value.
+ *
+ * State is maintained per device (identified by {@code Tablet.getDeviceId()} for tree-model or
+ * {@code Tablet.getTableName()} for table-model).
+ */
+public class ColumnAlignProcessor implements SubscriptionMessageProcessor {
+
+ // deviceKey -> (columnIndex -> lastValue)
+ private final Map> lastValues = new HashMap<>();
+
+ @Override
+ public List process(final List messages) {
+ for (final SubscriptionMessage message : messages) {
+ if (message.getMessageType() != SubscriptionMessageType.RECORD_HANDLER.getType()) {
+ continue;
+ }
+ final Iterator tablets = message.getRecordTabletIterator();
+ while (tablets.hasNext()) {
+ fillTablet(tablets.next());
+ }
+ }
+ return messages;
+ }
+
+ @Override
+ public List flush() {
+ return Collections.emptyList();
+ }
+
+ private void fillTablet(final Tablet tablet) {
+ final String deviceKey = getDeviceKey(tablet);
+ final Map cache = lastValues.computeIfAbsent(deviceKey, k -> new HashMap<>());
+
+ final Object[] values = tablet.getValues();
+ final BitMap[] bitMaps = tablet.getBitMaps();
+ final int rowSize = tablet.getRowSize();
+ final int columnCount = values.length;
+
+ for (int row = 0; row < rowSize; row++) {
+ for (int col = 0; col < columnCount; col++) {
+ final boolean isNull =
+ bitMaps != null && bitMaps[col] != null && bitMaps[col].isMarked(row);
+ if (isNull) {
+ // try forward-fill from cache
+ final Object cached = cache.get(col);
+ if (cached != null) {
+ setValueAt(values[col], row, cached);
+ bitMaps[col].unmark(row);
+ }
+ } else {
+ // update cache with this non-null value
+ cache.put(col, getValueAt(values[col], row));
+ }
+ }
+ }
+ }
+
+ private static String getDeviceKey(final Tablet tablet) {
+ // tree model uses deviceId; table model uses tableName
+ final String deviceId = tablet.getDeviceId();
+ return deviceId != null ? deviceId : tablet.getTableName();
+ }
+
+ private static Object getValueAt(final Object columnArray, final int row) {
+ if (columnArray instanceof long[]) {
+ return ((long[]) columnArray)[row];
+ } else if (columnArray instanceof int[]) {
+ return ((int[]) columnArray)[row];
+ } else if (columnArray instanceof double[]) {
+ return ((double[]) columnArray)[row];
+ } else if (columnArray instanceof float[]) {
+ return ((float[]) columnArray)[row];
+ } else if (columnArray instanceof boolean[]) {
+ return ((boolean[]) columnArray)[row];
+ } else if (columnArray instanceof Object[]) {
+ return ((Object[]) columnArray)[row];
+ }
+ return null;
+ }
+
+ private static void setValueAt(final Object columnArray, final int row, final Object value) {
+ if (columnArray instanceof long[]) {
+ ((long[]) columnArray)[row] = (Long) value;
+ } else if (columnArray instanceof int[]) {
+ ((int[]) columnArray)[row] = (Integer) value;
+ } else if (columnArray instanceof double[]) {
+ ((double[]) columnArray)[row] = (Double) value;
+ } else if (columnArray instanceof float[]) {
+ ((float[]) columnArray)[row] = (Float) value;
+ } else if (columnArray instanceof boolean[]) {
+ ((boolean[]) columnArray)[row] = (Boolean) value;
+ } else if (columnArray instanceof Object[]) {
+ ((Object[]) columnArray)[row] = value;
+ }
+ }
+}
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionMessageProcessor.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionMessageProcessor.java
new file mode 100644
index 0000000000000..ceee674cd6901
--- /dev/null
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionMessageProcessor.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.session.subscription.consumer.base;
+
+import org.apache.iotdb.session.subscription.payload.SubscriptionMessage;
+
+import java.util.List;
+
+/**
+ * A processor that transforms, filters, or enriches subscription messages in the pull consumer
+ * pipeline. Processors are chained and invoked on each poll() call.
+ *
+ * Processors may buffer messages internally (e.g., for watermark-based ordering) and return them
+ * in later process() calls. Buffered messages should be released via {@link #flush()} when the
+ * consumer closes.
+ */
+public interface SubscriptionMessageProcessor {
+
+ /**
+ * Process a batch of messages. May return fewer, more, or different messages than the input.
+ *
+ * @param messages the messages from the previous stage (or raw poll)
+ * @return messages to pass to the next stage (or to the user)
+ */
+ List process(List messages);
+
+ /**
+ * Flush all internally buffered messages. Called when the consumer is closing.
+ *
+ * @return any remaining buffered messages
+ */
+ List flush();
+
+ /** Returns the number of messages currently buffered by this processor. */
+ default int getBufferedCount() {
+ return 0;
+ }
+}
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/WatermarkProcessor.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/WatermarkProcessor.java
new file mode 100644
index 0000000000000..8c17896ce5de5
--- /dev/null
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/WatermarkProcessor.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.session.subscription.consumer.base;
+
+import org.apache.iotdb.session.subscription.payload.SubscriptionMessage;
+import org.apache.iotdb.session.subscription.payload.SubscriptionMessageType;
+
+import org.apache.tsfile.write.record.Tablet;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.PriorityQueue;
+
+/**
+ * A buffering processor that reorders messages based on watermark semantics. Messages are buffered
+ * internally and emitted only when the watermark advances past their maximum timestamp.
+ *
+ * Watermark = (minimum of latest timestamp per active source) - maxOutOfOrdernessMs
+ *
+ *
A source is considered "stale" if its latest timestamp has not increased for {@code
+ * staleSourceTimeoutMs}. Stale sources are excluded from the watermark calculation, preventing a
+ * single slow or idle source from anchoring the global watermark indefinitely.
+ *
+ *
Server-side WATERMARK events (carrying per-region timestamp progress) serve as heartbeats,
+ * confirming source liveness. They advance the per-source timestamp only when their timestamp is
+ * higher than the previously observed value.
+ *
+ *
A timeout mechanism ensures that buffered messages are eventually flushed even if no new data
+ * arrives, preventing unbounded buffering.
+ *
+ *
Note: This processor is primarily intended as a reference implementation. For
+ * production use with large-scale out-of-order data, consider using a downstream stream processing
+ * framework (Flink, Spark) for watermark handling.
+ */
+public class WatermarkProcessor implements SubscriptionMessageProcessor {
+
+ private static final long DEFAULT_STALE_SOURCE_TIMEOUT_MS = 30_000L;
+ private static final long DEFAULT_MAX_BUFFER_BYTES = 64L * 1024 * 1024; // 64 MB
+
+ private final long maxOutOfOrdernessMs;
+ private final long timeoutMs;
+ private final long staleSourceTimeoutMs;
+ private final long maxBufferBytes;
+
+ // Buffer ordered by message max timestamp
+ private final PriorityQueue buffer =
+ new PriorityQueue<>((a, b) -> Long.compare(a.maxTimestamp, b.maxTimestamp));
+
+ // Track latest timestamp per source (deviceId/tableName)
+ private final java.util.Map latestPerSource = new java.util.HashMap<>();
+ // Track wall-clock time when each source's timestamp last increased
+ private final java.util.Map lastAdvancedTimeMs = new java.util.HashMap<>();
+ private long lastEmitTimeMs = System.currentTimeMillis();
+ private long bufferedBytes = 0;
+
+ // Current watermark value
+ private long watermark = Long.MIN_VALUE;
+
+ /**
+ * Creates a WatermarkProcessor with default stale source timeout (30 seconds).
+ *
+ * @param maxOutOfOrdernessMs maximum expected out-of-orderness in milliseconds
+ * @param timeoutMs if no data arrives within this duration, force-flush all buffered messages
+ */
+ public WatermarkProcessor(final long maxOutOfOrdernessMs, final long timeoutMs) {
+ this(maxOutOfOrdernessMs, timeoutMs, DEFAULT_STALE_SOURCE_TIMEOUT_MS, DEFAULT_MAX_BUFFER_BYTES);
+ }
+
+ /**
+ * Creates a WatermarkProcessor.
+ *
+ * @param maxOutOfOrdernessMs maximum expected out-of-orderness in milliseconds
+ * @param timeoutMs if no data arrives within this duration, force-flush all buffered messages
+ * @param staleSourceTimeoutMs if a source's timestamp has not increased for this duration, it is
+ * excluded from watermark calculation. Use {@link Long#MAX_VALUE} to disable.
+ * @param maxBufferBytes maximum total estimated bytes of buffered messages. When exceeded, all
+ * buffered messages are force-flushed regardless of watermark. Defaults to 64 MB.
+ */
+ public WatermarkProcessor(
+ final long maxOutOfOrdernessMs,
+ final long timeoutMs,
+ final long staleSourceTimeoutMs,
+ final long maxBufferBytes) {
+ this.maxOutOfOrdernessMs = maxOutOfOrdernessMs;
+ this.timeoutMs = timeoutMs;
+ this.staleSourceTimeoutMs = staleSourceTimeoutMs;
+ this.maxBufferBytes = maxBufferBytes;
+ }
+
+ @Override
+ public List process(final List messages) {
+ final long now = System.currentTimeMillis();
+
+ // Buffer incoming messages and update per-source timestamps
+ for (final SubscriptionMessage message : messages) {
+ // WATERMARK events carry server-side timestamp progress per region.
+ // They serve as heartbeats and advance per-source tracking only when the timestamp
+ // actually increases.
+ if (message.getMessageType() == SubscriptionMessageType.WATERMARK.getType()) {
+ final String regionKey =
+ "region-"
+ + message.getCommitContext().getDataNodeId()
+ + "-"
+ + message.getCommitContext().getRegionId();
+ advanceSourceTimestamp(regionKey, message.getWatermarkTimestamp(), now);
+ continue; // Do not buffer system events
+ }
+
+ final long maxTs = extractMaxTimestamp(message);
+ final long estimatedSize = message.estimateSize();
+ buffer.add(new TimestampedMessage(message, maxTs, estimatedSize));
+ bufferedBytes += estimatedSize;
+ updateSourceTimestamp(message, maxTs, now);
+ }
+
+ // Compute watermark = min(latest per active source) - maxOutOfOrderness
+ // Sources whose timestamp has not increased for staleSourceTimeoutMs are excluded.
+ if (!latestPerSource.isEmpty()) {
+ long minLatest = Long.MAX_VALUE;
+ for (final java.util.Map.Entry entry : latestPerSource.entrySet()) {
+ final Long lastAdv = lastAdvancedTimeMs.get(entry.getKey());
+ if (lastAdv != null && (now - lastAdv) <= staleSourceTimeoutMs) {
+ minLatest = Math.min(minLatest, entry.getValue());
+ }
+ }
+ if (minLatest != Long.MAX_VALUE) {
+ watermark = minLatest - maxOutOfOrdernessMs;
+ }
+ // If all sources are stale, watermark stays unchanged and timeout will handle it.
+ }
+
+ // Emit messages whose maxTimestamp <= watermark
+ final List emitted = emit(watermark);
+
+ // Buffer overflow: force-flush all if buffer exceeds byte limit
+ if (bufferedBytes > maxBufferBytes) {
+ return forceFlushAll();
+ }
+
+ // Timeout: if nothing was emitted and timeout exceeded, force-flush all
+ if (emitted.isEmpty() && (now - lastEmitTimeMs) >= timeoutMs && !buffer.isEmpty()) {
+ return forceFlushAll();
+ }
+
+ if (!emitted.isEmpty()) {
+ lastEmitTimeMs = now;
+ }
+ return emitted;
+ }
+
+ @Override
+ public List flush() {
+ return forceFlushAll();
+ }
+
+ @Override
+ public int getBufferedCount() {
+ return buffer.size();
+ }
+
+ /** Returns the current watermark value. */
+ public long getWatermark() {
+ return watermark;
+ }
+
+ private List emit(final long watermarkValue) {
+ final List result = new ArrayList<>();
+ while (!buffer.isEmpty() && buffer.peek().maxTimestamp <= watermarkValue) {
+ final TimestampedMessage tm = buffer.poll();
+ bufferedBytes -= tm.estimatedSize;
+ result.add(tm.message);
+ }
+ return result;
+ }
+
+ private List forceFlushAll() {
+ final List result = new ArrayList<>(buffer.size());
+ while (!buffer.isEmpty()) {
+ result.add(buffer.poll().message);
+ }
+ bufferedBytes = 0;
+ lastEmitTimeMs = System.currentTimeMillis();
+ return result;
+ }
+
+ private static long extractMaxTimestamp(final SubscriptionMessage message) {
+ long maxTs = Long.MIN_VALUE;
+ if (message.getMessageType() == SubscriptionMessageType.RECORD_HANDLER.getType()) {
+ final Iterator it = message.getRecordTabletIterator();
+ while (it.hasNext()) {
+ final Tablet tablet = it.next();
+ final long[] timestamps = tablet.getTimestamps();
+ final int rowSize = tablet.getRowSize();
+ for (int i = 0; i < rowSize; i++) {
+ maxTs = Math.max(maxTs, timestamps[i]);
+ }
+ }
+ }
+ // For non-tablet messages or empty messages, use current wall clock
+ if (maxTs == Long.MIN_VALUE) {
+ maxTs = System.currentTimeMillis();
+ }
+ return maxTs;
+ }
+
+ private void updateSourceTimestamp(
+ final SubscriptionMessage message, final long maxTs, final long nowMs) {
+ // Use region-based key so data events and WATERMARK events share the same key namespace.
+ final String regionId = message.getCommitContext().getRegionId();
+ final int dataNodeId = message.getCommitContext().getDataNodeId();
+ final String key = "region-" + dataNodeId + "-" + regionId;
+ advanceSourceTimestamp(key, maxTs, nowMs);
+ }
+
+ /**
+ * Updates the per-source timestamp tracking. Only records a new "last advanced" wall-clock time
+ * when the timestamp actually increases, so that stale sources (whose timestamps don't advance)
+ * are eventually excluded from watermark calculation.
+ */
+ private void advanceSourceTimestamp(final String key, final long newTs, final long nowMs) {
+ final Long oldTs = latestPerSource.get(key);
+ if (oldTs == null || newTs > oldTs) {
+ latestPerSource.put(key, newTs);
+ lastAdvancedTimeMs.put(key, nowMs);
+ }
+ }
+
+ private static final class TimestampedMessage {
+ final SubscriptionMessage message;
+ final long maxTimestamp;
+ final long estimatedSize;
+
+ TimestampedMessage(
+ final SubscriptionMessage message, final long maxTimestamp, final long estimatedSize) {
+ this.message = message;
+ this.maxTimestamp = maxTimestamp;
+ this.estimatedSize = estimatedSize;
+ }
+ }
+}
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/table/SubscriptionTablePullConsumer.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/table/SubscriptionTablePullConsumer.java
index 83dd39aebbf7d..e3fb90cda470a 100644
--- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/table/SubscriptionTablePullConsumer.java
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/table/SubscriptionTablePullConsumer.java
@@ -25,6 +25,8 @@
import org.apache.iotdb.session.subscription.consumer.ISubscriptionTablePullConsumer;
import org.apache.iotdb.session.subscription.consumer.base.AbstractSubscriptionProvider;
import org.apache.iotdb.session.subscription.consumer.base.AbstractSubscriptionPullConsumer;
+import org.apache.iotdb.session.subscription.consumer.base.SubscriptionMessageProcessor;
+import org.apache.iotdb.session.subscription.payload.PollResult;
import org.apache.iotdb.session.subscription.payload.SubscriptionMessage;
import java.time.Duration;
@@ -173,4 +175,24 @@ public String getConsumerGroupId() {
public boolean allTopicMessagesHaveBeenConsumed() {
return super.allTopicMessagesHaveBeenConsumed();
}
+
+ /////////////////////////////// processor ///////////////////////////////
+
+ public SubscriptionTablePullConsumer addProcessor(final SubscriptionMessageProcessor processor) {
+ super.addProcessor(processor);
+ return this;
+ }
+
+ public PollResult pollWithInfo(final long timeoutMs) throws SubscriptionException {
+ return super.pollWithInfo(timeoutMs);
+ }
+
+ public PollResult pollWithInfo(final Duration timeout) throws SubscriptionException {
+ return super.pollWithInfo(timeout.toMillis());
+ }
+
+ public PollResult pollWithInfo(final Set topicNames, final long timeoutMs)
+ throws SubscriptionException {
+ return super.pollWithInfo(topicNames, timeoutMs);
+ }
}
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/tree/SubscriptionTreePullConsumer.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/tree/SubscriptionTreePullConsumer.java
index 23050893f660d..c4daab68839aa 100644
--- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/tree/SubscriptionTreePullConsumer.java
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/tree/SubscriptionTreePullConsumer.java
@@ -27,6 +27,8 @@
import org.apache.iotdb.session.subscription.consumer.ISubscriptionTreePullConsumer;
import org.apache.iotdb.session.subscription.consumer.base.AbstractSubscriptionProvider;
import org.apache.iotdb.session.subscription.consumer.base.AbstractSubscriptionPullConsumer;
+import org.apache.iotdb.session.subscription.consumer.base.SubscriptionMessageProcessor;
+import org.apache.iotdb.session.subscription.payload.PollResult;
import org.apache.iotdb.session.subscription.payload.SubscriptionMessage;
import org.apache.iotdb.session.subscription.util.IdentifierUtils;
@@ -220,6 +222,26 @@ public boolean allTopicMessagesHaveBeenConsumed() {
return super.allTopicMessagesHaveBeenConsumed();
}
+ /////////////////////////////// processor ///////////////////////////////
+
+ public SubscriptionTreePullConsumer addProcessor(final SubscriptionMessageProcessor processor) {
+ super.addProcessor(processor);
+ return this;
+ }
+
+ public PollResult pollWithInfo(final long timeoutMs) throws SubscriptionException {
+ return super.pollWithInfo(timeoutMs);
+ }
+
+ public PollResult pollWithInfo(final Duration timeout) throws SubscriptionException {
+ return super.pollWithInfo(timeout.toMillis());
+ }
+
+ public PollResult pollWithInfo(final Set topicNames, final long timeoutMs)
+ throws SubscriptionException {
+ return super.pollWithInfo(topicNames, timeoutMs);
+ }
+
/////////////////////////////// builder ///////////////////////////////
@Deprecated // keep for forward compatibility
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/PollResult.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/PollResult.java
new file mode 100644
index 0000000000000..be56548116e11
--- /dev/null
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/PollResult.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.session.subscription.payload;
+
+import java.util.Collections;
+import java.util.List;
+
+/** Result of a poll operation that includes processor metadata alongside the messages. */
+public class PollResult {
+
+ private final List messages;
+ private final int bufferedCount;
+ private final long watermark;
+
+ public PollResult(
+ final List messages, final int bufferedCount, final long watermark) {
+ this.messages = messages != null ? messages : Collections.emptyList();
+ this.bufferedCount = bufferedCount;
+ this.watermark = watermark;
+ }
+
+ /** Returns the processed messages ready for consumption. */
+ public List getMessages() {
+ return messages;
+ }
+
+ /** Returns the total number of messages currently buffered across all processors. */
+ public int getBufferedCount() {
+ return bufferedCount;
+ }
+
+ /**
+ * Returns the current watermark timestamp (-1 if no watermark processor is configured). Messages
+ * with timestamps at or before this value have all been emitted.
+ */
+ public long getWatermark() {
+ return watermark;
+ }
+
+ @Override
+ public String toString() {
+ return "PollResult{messages="
+ + messages.size()
+ + ", bufferedCount="
+ + bufferedCount
+ + ", watermark="
+ + watermark
+ + "}";
+ }
+}
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionMessage.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionMessage.java
index f41d34f0ab83a..9ba61f6bc2718 100644
--- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionMessage.java
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionMessage.java
@@ -39,11 +39,15 @@ public class SubscriptionMessage implements Comparable {
private final SubscriptionMessageHandler handler;
+ /** Watermark timestamp, valid only when messageType == WATERMARK. */
+ private final long watermarkTimestamp;
+
public SubscriptionMessage(
final SubscriptionCommitContext commitContext, final Map> tablets) {
this.commitContext = commitContext;
this.messageType = SubscriptionMessageType.RECORD_HANDLER.getType();
this.handler = new SubscriptionRecordHandler(tablets);
+ this.watermarkTimestamp = Long.MIN_VALUE;
}
public SubscriptionMessage(
@@ -53,6 +57,16 @@ public SubscriptionMessage(
this.commitContext = commitContext;
this.messageType = SubscriptionMessageType.TS_FILE.getType();
this.handler = new SubscriptionTsFileHandler(absolutePath, databaseName);
+ this.watermarkTimestamp = Long.MIN_VALUE;
+ }
+
+ /** Watermark message carrying server-side timestamp progress for a region. */
+ public SubscriptionMessage(
+ final SubscriptionCommitContext commitContext, final long watermarkTimestamp) {
+ this.commitContext = commitContext;
+ this.messageType = SubscriptionMessageType.WATERMARK.getType();
+ this.handler = null;
+ this.watermarkTimestamp = watermarkTimestamp;
}
public SubscriptionCommitContext getCommitContext() {
@@ -63,6 +77,34 @@ public short getMessageType() {
return messageType;
}
+ /**
+ * Returns the watermark timestamp carried by this message. Only valid when {@code
+ * getMessageType() == SubscriptionMessageType.WATERMARK.getType()}.
+ *
+ * @return the watermark timestamp, or {@code Long.MIN_VALUE} if not a watermark message
+ */
+ public long getWatermarkTimestamp() {
+ return watermarkTimestamp;
+ }
+
+ /**
+ * Estimates the heap memory occupied by this message in bytes. For tablet-based messages, this
+ * delegates to {@link Tablet#ramBytesUsed()} for accurate per-column estimation.
+ *
+ * @return estimated byte size
+ */
+ public long estimateSize() {
+ // Object header + references + primitives (rough constant)
+ long size = 64;
+ if (handler instanceof SubscriptionRecordHandler) {
+ final Iterator it = getRecordTabletIterator();
+ while (it.hasNext()) {
+ size += it.next().ramBytesUsed();
+ }
+ }
+ return size;
+ }
+
/////////////////////////////// override ///////////////////////////////
@Override
@@ -75,13 +117,14 @@ public boolean equals(final Object obj) {
}
final SubscriptionMessage that = (SubscriptionMessage) obj;
return Objects.equals(this.commitContext, that.commitContext)
+ && this.watermarkTimestamp == that.watermarkTimestamp
&& Objects.equals(this.messageType, that.messageType)
&& Objects.equals(this.handler, that.handler);
}
@Override
public int hashCode() {
- return Objects.hash(commitContext, messageType, handler);
+ return Objects.hash(commitContext, messageType, handler, watermarkTimestamp);
}
@Override
@@ -95,6 +138,8 @@ public String toString() {
+ commitContext
+ ", messageType="
+ SubscriptionMessageType.valueOf(messageType).toString()
+ + ", watermarkTimestamp="
+ + watermarkTimestamp
+ "}";
}
diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionMessageType.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionMessageType.java
index 34189c2fa9b42..0732c0590c181 100644
--- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionMessageType.java
+++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionMessageType.java
@@ -26,6 +26,7 @@
public enum SubscriptionMessageType {
RECORD_HANDLER((short) 0),
TS_FILE((short) 1),
+ WATERMARK((short) 3),
;
private final short type;
diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionCommitContextTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionCommitContextTest.java
new file mode 100644
index 0000000000000..c1c143447cc1d
--- /dev/null
+++ b/iotdb-client/subscription/src/test/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionCommitContextTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.rpc.subscription.payload.poll;
+
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+import org.junit.Test;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class SubscriptionCommitContextTest {
+
+ @Test
+ public void testDeserializeV1Compatibility() throws IOException {
+ final ByteBuffer buffer = buildV1Buffer(1, 2, "topic", "group", 3L);
+
+ final SubscriptionCommitContext context = SubscriptionCommitContext.deserialize(buffer);
+
+ assertEquals(1, context.getDataNodeId());
+ assertEquals(2, context.getRebootTimes());
+ assertEquals("topic", context.getTopicName());
+ assertEquals("group", context.getConsumerGroupId());
+ assertEquals(3L, context.getCommitId());
+ assertEquals(0L, context.getSeekGeneration());
+ assertEquals("", context.getRegionId());
+ assertEquals(0L, context.getPhysicalTime());
+ assertFalse(context.hasWriterProgress());
+ assertTrue(context.hasLegacyCommitId());
+ assertTrue(context.isCommittable());
+ }
+
+ @Test
+ public void testDeserializeV2() throws IOException {
+ final SubscriptionCommitContext original =
+ new SubscriptionCommitContext(1, 2, "topic", "group", 3L, 4L, "region", 5L);
+
+ final ByteBuffer buffer = SubscriptionCommitContext.serialize(original);
+ final SubscriptionCommitContext parsed = SubscriptionCommitContext.deserialize(buffer);
+
+ assertEquals(original, parsed);
+ assertFalse(parsed.hasWriterProgress());
+ assertTrue(parsed.hasLegacyCommitId());
+ }
+
+ @Test
+ public void testDeserializeV3() throws IOException {
+ final WriterId writerId = new WriterId("region", 7, 8L);
+ final WriterProgress writerProgress = new WriterProgress(9L, 10L);
+ final SubscriptionCommitContext original =
+ new SubscriptionCommitContext(1, 2, "topic", "group", 3L, writerId, writerProgress);
+
+ final ByteBuffer buffer = SubscriptionCommitContext.serialize(original);
+ final SubscriptionCommitContext parsed = SubscriptionCommitContext.deserialize(buffer);
+
+ assertEquals(original, parsed);
+ assertEquals(writerId, parsed.getWriterId());
+ assertEquals(writerProgress, parsed.getWriterProgress());
+ assertEquals("region", parsed.getRegionId());
+ assertEquals(9L, parsed.getPhysicalTime());
+ assertEquals(10L, parsed.getLocalSeq());
+ assertTrue(parsed.hasWriterProgress());
+ assertFalse(parsed.hasLegacyCommitId());
+ assertTrue(parsed.isCommittable());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testDeserializeUnsupportedVersion() throws IOException {
+ final ByteBuffer buffer = buildV1BufferWithVersion((byte) 4, 1, 2, "topic", "group", 3L);
+ SubscriptionCommitContext.deserialize(buffer);
+ }
+
+ private static ByteBuffer buildV1Buffer(
+ final int dataNodeId,
+ final int rebootTimes,
+ final String topicName,
+ final String consumerGroupId,
+ final long commitId)
+ throws IOException {
+ return buildV1BufferWithVersion(
+ (byte) 1, dataNodeId, rebootTimes, topicName, consumerGroupId, commitId);
+ }
+
+ private static ByteBuffer buildV1BufferWithVersion(
+ final byte version,
+ final int dataNodeId,
+ final int rebootTimes,
+ final String topicName,
+ final String consumerGroupId,
+ final long commitId)
+ throws IOException {
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(version, outputStream);
+ ReadWriteIOUtils.write(dataNodeId, outputStream);
+ ReadWriteIOUtils.write(rebootTimes, outputStream);
+ ReadWriteIOUtils.write(topicName, outputStream);
+ ReadWriteIOUtils.write(consumerGroupId, outputStream);
+ ReadWriteIOUtils.write(commitId, outputStream);
+ return ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+ }
+}
diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollRequestTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollRequestTest.java
new file mode 100644
index 0000000000000..ecfea3d160bc4
--- /dev/null
+++ b/iotdb-client/subscription/src/test/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollRequestTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.rpc.subscription.payload.poll;
+
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class SubscriptionPollRequestTest {
+
+ @Test
+ public void testRoundTripWithProgressByTopic() throws IOException {
+ final Map writerPositions = new LinkedHashMap<>();
+ writerPositions.put(new WriterId("1_100", 7, 2L), new WriterProgress(1001L, 11L));
+ writerPositions.put(new WriterId("1_100", 8, 1L), new WriterProgress(999L, 9L));
+
+ final TopicProgress topicProgress =
+ new TopicProgress(Collections.singletonMap("1_100", new RegionProgress(writerPositions)));
+ final Map progressByTopic = new LinkedHashMap<>();
+ progressByTopic.put("topicA", topicProgress);
+
+ final SubscriptionPollRequest original =
+ new SubscriptionPollRequest(
+ SubscriptionPollRequestType.POLL.getType(),
+ new PollPayload(Collections.singleton("topicA")),
+ 1234L,
+ 4096L,
+ progressByTopic);
+
+ final ByteBuffer serialized = SubscriptionPollRequest.serialize(original);
+ final SubscriptionPollRequest parsed = SubscriptionPollRequest.deserialize(serialized);
+
+ assertEquals(original.getRequestType(), parsed.getRequestType());
+ assertEquals(original.getTimeoutMs(), parsed.getTimeoutMs());
+ assertEquals(original.getMaxBytes(), parsed.getMaxBytes());
+ assertEquals(original.getPayload(), parsed.getPayload());
+ assertEquals(progressByTopic, parsed.getProgressByTopic());
+ }
+}
diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/rpc/subscription/payload/request/PipeSubscribeSeekReqTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/rpc/subscription/payload/request/PipeSubscribeSeekReqTest.java
new file mode 100644
index 0000000000000..c2afb43110289
--- /dev/null
+++ b/iotdb-client/subscription/src/test/java/org/apache/iotdb/rpc/subscription/payload/request/PipeSubscribeSeekReqTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.rpc.subscription.payload.request;
+
+import org.apache.iotdb.rpc.subscription.payload.poll.RegionProgress;
+import org.apache.iotdb.rpc.subscription.payload.poll.TopicProgress;
+import org.apache.iotdb.rpc.subscription.payload.poll.WriterId;
+import org.apache.iotdb.rpc.subscription.payload.poll.WriterProgress;
+
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class PipeSubscribeSeekReqTest {
+
+ @Test
+ public void testTopicProgressSeekRoundTrip() throws IOException {
+ final Map writerPositions = new LinkedHashMap<>();
+ writerPositions.put(new WriterId("1_100", 1, 2L), new WriterProgress(1000L, 10L));
+ final TopicProgress original =
+ new TopicProgress(Collections.singletonMap("1_100", new RegionProgress(writerPositions)));
+
+ final PipeSubscribeSeekReq req =
+ PipeSubscribeSeekReq.toTPipeSubscribeSeekAfterReq("topicA", original);
+ final PipeSubscribeSeekReq parsed = PipeSubscribeSeekReq.fromTPipeSubscribeReq(req);
+
+ assertEquals(PipeSubscribeSeekReq.SEEK_AFTER_TOPIC_PROGRESS, parsed.getSeekType());
+ assertEquals("topicA", parsed.getTopicName());
+ assertEquals(original, parsed.getTopicProgress());
+ }
+}
diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumerProgressTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumerProgressTest.java
new file mode 100644
index 0000000000000..726aab5666866
--- /dev/null
+++ b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumerProgressTest.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.session.subscription.consumer.base;
+
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.rpc.subscription.payload.poll.RegionProgress;
+import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext;
+import org.apache.iotdb.rpc.subscription.payload.poll.TopicProgress;
+import org.apache.iotdb.rpc.subscription.payload.poll.WriterId;
+import org.apache.iotdb.rpc.subscription.payload.poll.WriterProgress;
+import org.apache.iotdb.session.subscription.payload.SubscriptionMessage;
+
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class AbstractSubscriptionConsumerProgressTest {
+
+ private static final String TOPIC = "topic_progress_test";
+ private static final String GROUP = "group_progress_test";
+ private static final String REGION = "1_100";
+
+ @Test
+ public void testAdvanceCurrentPositionsWithWriterProgress() throws Exception {
+ final TestSubscriptionConsumer consumer = newConsumer();
+ final WriterId writerId = new WriterId(REGION, 1, 2L);
+ final WriterProgress writerProgress = new WriterProgress(100L, 10L);
+ final SubscriptionMessage message =
+ new SubscriptionMessage(
+ new SubscriptionCommitContext(1, 0, TOPIC, GROUP, 0L, writerId, writerProgress),
+ Collections.emptyMap());
+
+ invokeAdvanceCurrentPositions(consumer, Collections.singletonList(message));
+
+ final TopicProgress positions = consumer.positions(TOPIC);
+ assertEquals(
+ writerProgress,
+ positions.getRegionProgress().get(REGION).getWriterPositions().get(writerId));
+ }
+
+ @Test
+ public void testAdvanceCommittedPositionsFallsBackToLegacyFields() throws Exception {
+ final TestSubscriptionConsumer consumer = newConsumer();
+ final SubscriptionCommitContext legacyContext =
+ new SubscriptionCommitContext(7, 0, TOPIC, GROUP, 11L, 0L, REGION, 101L);
+
+ invokeAdvanceCommittedPositions(consumer, Collections.singletonList(legacyContext));
+
+ final TopicProgress committed = consumer.committedPositions(TOPIC);
+ final RegionProgress regionProgress = committed.getRegionProgress().get(REGION);
+ assertNotNull(regionProgress);
+ assertEquals(1, regionProgress.getWriterPositions().size());
+ final Map.Entry onlyEntry =
+ regionProgress.getWriterPositions().entrySet().iterator().next();
+ assertEquals(new WriterId(REGION, 7, 0L), onlyEntry.getKey());
+ assertEquals(new WriterProgress(101L, 11L), onlyEntry.getValue());
+ }
+
+ @Test
+ public void testAdvanceCurrentPositionsMergesPerWriterAndKeepsNewest() throws Exception {
+ final TestSubscriptionConsumer consumer = newConsumer();
+
+ final WriterId writer1 = new WriterId(REGION, 1, 1L);
+ final WriterId writer2 = new WriterId(REGION, 2, 1L);
+ final SubscriptionMessage olderWriter1 =
+ new SubscriptionMessage(
+ new SubscriptionCommitContext(
+ 1, 0, TOPIC, GROUP, 0L, writer1, new WriterProgress(100L, 8L)),
+ Collections.emptyMap());
+ final SubscriptionMessage newerWriter1 =
+ new SubscriptionMessage(
+ new SubscriptionCommitContext(
+ 1, 0, TOPIC, GROUP, 0L, writer1, new WriterProgress(100L, 10L)),
+ Collections.emptyMap());
+ final SubscriptionMessage writer2Message =
+ new SubscriptionMessage(
+ new SubscriptionCommitContext(
+ 2, 0, TOPIC, GROUP, 0L, writer2, new WriterProgress(95L, 7L)),
+ Collections.emptyMap());
+
+ invokeAdvanceCurrentPositions(
+ consumer, Arrays.asList(olderWriter1, newerWriter1, writer2Message));
+
+ final RegionProgress regionProgress = consumer.positions(TOPIC).getRegionProgress().get(REGION);
+ assertEquals(2, regionProgress.getWriterPositions().size());
+ assertEquals(new WriterProgress(100L, 10L), regionProgress.getWriterPositions().get(writer1));
+ assertEquals(new WriterProgress(95L, 7L), regionProgress.getWriterPositions().get(writer2));
+ }
+
+ private static TestSubscriptionConsumer newConsumer() throws Exception {
+ final TestSubscriptionConsumer consumer =
+ new TestSubscriptionConsumer(
+ new AbstractSubscriptionConsumerBuilder()
+ .consumerId("progress_consumer")
+ .consumerGroupId(GROUP));
+ final Field isClosedField = AbstractSubscriptionConsumer.class.getDeclaredField("isClosed");
+ isClosedField.setAccessible(true);
+ ((AtomicBoolean) isClosedField.get(consumer)).set(false);
+ return consumer;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static void invokeAdvanceCurrentPositions(
+ final AbstractSubscriptionConsumer consumer, final List messages)
+ throws Exception {
+ final Method method =
+ AbstractSubscriptionConsumer.class.getDeclaredMethod("advanceCurrentPositions", List.class);
+ method.setAccessible(true);
+ method.invoke(consumer, messages);
+ }
+
+ private static void invokeAdvanceCommittedPositions(
+ final AbstractSubscriptionConsumer consumer,
+ final List commitContexts)
+ throws Exception {
+ final Method method =
+ AbstractSubscriptionConsumer.class.getDeclaredMethod(
+ "advanceCommittedPositions", List.class);
+ method.setAccessible(true);
+ method.invoke(consumer, commitContexts);
+ }
+
+ private static final class TestSubscriptionConsumer extends AbstractSubscriptionConsumer {
+
+ private TestSubscriptionConsumer(final AbstractSubscriptionConsumerBuilder builder) {
+ super(builder);
+ }
+
+ @Override
+ protected AbstractSubscriptionProvider constructSubscriptionProvider(
+ final TEndPoint endPoint,
+ final String username,
+ final String password,
+ final String consumerId,
+ final String consumerGroupId,
+ final int thriftMaxFrameSize,
+ final long heartbeatIntervalMs,
+ final int connectionTimeoutInMs) {
+ throw new UnsupportedOperationException("No provider needed for progress unit tests");
+ }
+ }
+}
diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumerSeekProgressTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumerSeekProgressTest.java
new file mode 100644
index 0000000000000..97dad18bf9aea
--- /dev/null
+++ b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumerSeekProgressTest.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.session.subscription.consumer.base;
+
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.rpc.subscription.payload.poll.RegionProgress;
+import org.apache.iotdb.rpc.subscription.payload.poll.TopicProgress;
+import org.apache.iotdb.rpc.subscription.payload.poll.WriterId;
+import org.apache.iotdb.rpc.subscription.payload.poll.WriterProgress;
+
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class AbstractSubscriptionConsumerSeekProgressTest {
+
+ private static final String TOPIC = "topic_seek_progress_test";
+ private static final String REGION_A = "1_100";
+ private static final String REGION_B = "1_101";
+
+ @Test
+ public void testOverlayTopicProgressPreservesMissingRegions() throws Exception {
+ final TestSubscriptionConsumer consumer = newConsumer();
+
+ invokeSetCurrentPositions(
+ consumer,
+ TOPIC,
+ buildTopicProgress(
+ REGION_A,
+ new WriterId(REGION_A, 1, 1L),
+ new WriterProgress(100L, 10L),
+ REGION_B,
+ new WriterId(REGION_B, 2, 1L),
+ new WriterProgress(200L, 20L)));
+
+ invokeOverlayCurrentPositions(
+ consumer,
+ TOPIC,
+ new TopicProgress(
+ Collections.singletonMap(
+ REGION_A,
+ new RegionProgress(
+ Collections.singletonMap(
+ new WriterId(REGION_A, 1, 1L), new WriterProgress(50L, 5L))))));
+
+ final TopicProgress positions = consumer.positions(TOPIC);
+ assertNotNull(positions.getRegionProgress().get(REGION_A));
+ assertNotNull(positions.getRegionProgress().get(REGION_B));
+ assertEquals(
+ new WriterProgress(50L, 5L),
+ positions
+ .getRegionProgress()
+ .get(REGION_A)
+ .getWriterPositions()
+ .values()
+ .iterator()
+ .next());
+ assertEquals(
+ new WriterProgress(200L, 20L),
+ positions
+ .getRegionProgress()
+ .get(REGION_B)
+ .getWriterPositions()
+ .values()
+ .iterator()
+ .next());
+ }
+
+ @Test
+ public void testOverlayTopicProgressAllowsSeekBackwardsForSpecifiedRegion() throws Exception {
+ final TestSubscriptionConsumer consumer = newConsumer();
+
+ invokeSetCommittedPositions(
+ consumer,
+ TOPIC,
+ new TopicProgress(
+ Collections.singletonMap(
+ REGION_A,
+ new RegionProgress(
+ Collections.singletonMap(
+ new WriterId(REGION_A, 1, 1L), new WriterProgress(100L, 10L))))));
+
+ invokeOverlayCommittedPositions(
+ consumer,
+ TOPIC,
+ new TopicProgress(
+ Collections.singletonMap(
+ REGION_A,
+ new RegionProgress(
+ Collections.singletonMap(
+ new WriterId(REGION_A, 1, 1L), new WriterProgress(80L, 4L))))));
+
+ assertEquals(
+ new WriterProgress(80L, 4L),
+ consumer
+ .committedPositions(TOPIC)
+ .getRegionProgress()
+ .get(REGION_A)
+ .getWriterPositions()
+ .values()
+ .iterator()
+ .next());
+ }
+
+ private static TestSubscriptionConsumer newConsumer() throws Exception {
+ final TestSubscriptionConsumer consumer =
+ new TestSubscriptionConsumer(
+ new AbstractSubscriptionConsumerBuilder()
+ .consumerId("seek_progress_consumer")
+ .consumerGroupId("seek_progress_group"));
+ final Field isClosedField = AbstractSubscriptionConsumer.class.getDeclaredField("isClosed");
+ isClosedField.setAccessible(true);
+ ((AtomicBoolean) isClosedField.get(consumer)).set(false);
+ return consumer;
+ }
+
+ private static void invokeSetCurrentPositions(
+ final AbstractSubscriptionConsumer consumer,
+ final String topicName,
+ final TopicProgress topicProgress)
+ throws Exception {
+ final Method method =
+ AbstractSubscriptionConsumer.class.getDeclaredMethod(
+ "setCurrentPositions", String.class, TopicProgress.class);
+ method.setAccessible(true);
+ method.invoke(consumer, topicName, topicProgress);
+ }
+
+ private static void invokeSetCommittedPositions(
+ final AbstractSubscriptionConsumer consumer,
+ final String topicName,
+ final TopicProgress topicProgress)
+ throws Exception {
+ final Method method =
+ AbstractSubscriptionConsumer.class.getDeclaredMethod(
+ "setCommittedPositions", String.class, TopicProgress.class);
+ method.setAccessible(true);
+ method.invoke(consumer, topicName, topicProgress);
+ }
+
+ private static void invokeOverlayCurrentPositions(
+ final AbstractSubscriptionConsumer consumer,
+ final String topicName,
+ final TopicProgress topicProgress)
+ throws Exception {
+ final Method method =
+ AbstractSubscriptionConsumer.class.getDeclaredMethod(
+ "overlayCurrentPositions", String.class, TopicProgress.class);
+ method.setAccessible(true);
+ method.invoke(consumer, topicName, topicProgress);
+ }
+
+ private static void invokeOverlayCommittedPositions(
+ final AbstractSubscriptionConsumer consumer,
+ final String topicName,
+ final TopicProgress topicProgress)
+ throws Exception {
+ final Method method =
+ AbstractSubscriptionConsumer.class.getDeclaredMethod(
+ "overlayCommittedPositions", String.class, TopicProgress.class);
+ method.setAccessible(true);
+ method.invoke(consumer, topicName, topicProgress);
+ }
+
+ private static TopicProgress buildTopicProgress(
+ final String regionA,
+ final WriterId writerA,
+ final WriterProgress writerProgressA,
+ final String regionB,
+ final WriterId writerB,
+ final WriterProgress writerProgressB) {
+ final Map regionProgress = new LinkedHashMap<>();
+ regionProgress.put(
+ regionA, new RegionProgress(Collections.singletonMap(writerA, writerProgressA)));
+ regionProgress.put(
+ regionB, new RegionProgress(Collections.singletonMap(writerB, writerProgressB)));
+ return new TopicProgress(regionProgress);
+ }
+
+ private static final class TestSubscriptionConsumer extends AbstractSubscriptionConsumer {
+
+ private TestSubscriptionConsumer(final AbstractSubscriptionConsumerBuilder builder) {
+ super(builder);
+ }
+
+ @Override
+ protected AbstractSubscriptionProvider constructSubscriptionProvider(
+ final TEndPoint endPoint,
+ final String username,
+ final String password,
+ final String consumerId,
+ final String consumerGroupId,
+ final int thriftMaxFrameSize,
+ final long heartbeatIntervalMs,
+ final int connectionTimeoutInMs) {
+ throw new UnsupportedOperationException("No provider needed for seek progress unit tests");
+ }
+ }
+}
diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/WatermarkProcessorTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/WatermarkProcessorTest.java
new file mode 100644
index 0000000000000..613090650bd1a
--- /dev/null
+++ b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/WatermarkProcessorTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.session.subscription.consumer.base;
+
+import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext;
+import org.apache.iotdb.session.subscription.payload.SubscriptionMessage;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+public class WatermarkProcessorTest {
+
+ private static final String TOPIC = "topic1";
+ private static final String GROUP = "group1";
+ private static final String REGION_R1 = "R1";
+ private static final String REGION_R2 = "R2";
+
+ private static SubscriptionMessage dataMsg(final String regionId, final int dataNodeId) {
+ final SubscriptionCommitContext ctx =
+ new SubscriptionCommitContext(dataNodeId, 0, TOPIC, GROUP, 0L, 0L, regionId, 0L);
+ return new SubscriptionMessage(ctx, Collections.emptyMap());
+ }
+
+ private static SubscriptionMessage watermarkMsg(
+ final String regionId, final int dataNodeId, final long watermarkTs) {
+ final SubscriptionCommitContext ctx =
+ new SubscriptionCommitContext(dataNodeId, 0, TOPIC, GROUP, 0L, 0L, regionId, 0L);
+ return new SubscriptionMessage(ctx, watermarkTs);
+ }
+
+ @Test
+ public void testSingleRegionRelease() {
+ final WatermarkProcessor proc = new WatermarkProcessor(5, 60_000);
+
+ final List result =
+ proc.process(Collections.singletonList(watermarkMsg(REGION_R1, 1, 1000)));
+
+ Assert.assertTrue(result.isEmpty());
+ Assert.assertEquals(995, proc.getWatermark());
+ }
+
+ @Test
+ public void testTwoRegionsMinWatermark() {
+ final WatermarkProcessor proc = new WatermarkProcessor(10, 60_000);
+
+ proc.process(Arrays.asList(watermarkMsg(REGION_R1, 1, 2000), watermarkMsg(REGION_R2, 1, 500)));
+
+ Assert.assertEquals(490, proc.getWatermark());
+ }
+
+ @Test
+ public void testWatermarkAdvancesIdleRegion() {
+ final WatermarkProcessor proc = new WatermarkProcessor(5, 60_000);
+
+ proc.process(Arrays.asList(watermarkMsg(REGION_R1, 1, 2000), watermarkMsg(REGION_R2, 1, 500)));
+ Assert.assertEquals(495, proc.getWatermark());
+
+ proc.process(Collections.singletonList(watermarkMsg(REGION_R2, 1, 1500)));
+ Assert.assertEquals(1495, proc.getWatermark());
+
+ proc.process(Collections.singletonList(watermarkMsg(REGION_R2, 1, 3000)));
+ Assert.assertEquals(1995, proc.getWatermark());
+ }
+
+ @Test
+ public void testWatermarkEventsNotBuffered() {
+ final WatermarkProcessor proc = new WatermarkProcessor(5, 60_000);
+
+ proc.process(Collections.singletonList(watermarkMsg(REGION_R1, 1, 1000)));
+
+ Assert.assertEquals(0, proc.getBufferedCount());
+ }
+
+ @Test
+ public void testFlushReleasesAll() {
+ final WatermarkProcessor proc = new WatermarkProcessor(5, 60_000);
+
+ proc.process(Arrays.asList(dataMsg(REGION_R1, 1), dataMsg(REGION_R1, 1)));
+
+ proc.flush();
+ Assert.assertEquals(0, proc.getBufferedCount());
+ }
+
+ @Test
+ public void testWatermarkNoRegression() {
+ final WatermarkProcessor proc = new WatermarkProcessor(10, 60_000);
+
+ proc.process(Collections.singletonList(watermarkMsg(REGION_R1, 1, 2000)));
+ Assert.assertEquals(1990, proc.getWatermark());
+
+ proc.process(Collections.singletonList(watermarkMsg(REGION_R1, 1, 1500)));
+ Assert.assertEquals(1990, proc.getWatermark());
+ }
+
+ @Test
+ public void testMultipleWatermarksInSingleBatch() {
+ final WatermarkProcessor proc = new WatermarkProcessor(0, 60_000);
+
+ proc.process(
+ Arrays.asList(
+ watermarkMsg(REGION_R1, 1, 100),
+ watermarkMsg(REGION_R2, 1, 200),
+ watermarkMsg(REGION_R1, 1, 300)));
+
+ Assert.assertEquals(200, proc.getWatermark());
+ }
+
+ @Test
+ public void testEmptyInput() {
+ final WatermarkProcessor proc = new WatermarkProcessor(5, 60_000);
+
+ final List result = proc.process(Collections.emptyList());
+ Assert.assertTrue(result.isEmpty());
+ Assert.assertEquals(Long.MIN_VALUE, proc.getWatermark());
+ }
+
+ @Test
+ public void testThreeRegionsSlowestDeterminesWatermark() {
+ final WatermarkProcessor proc = new WatermarkProcessor(10, 60_000);
+
+ proc.process(
+ Arrays.asList(
+ watermarkMsg(REGION_R1, 1, 5000),
+ watermarkMsg(REGION_R2, 1, 3000),
+ watermarkMsg("R3", 2, 4000)));
+
+ Assert.assertEquals(2990, proc.getWatermark());
+
+ proc.process(Collections.singletonList(watermarkMsg(REGION_R2, 1, 6000)));
+ Assert.assertEquals(3990, proc.getWatermark());
+ }
+
+ @Test
+ public void testZeroOutOfOrderness() {
+ final WatermarkProcessor proc = new WatermarkProcessor(0, 60_000);
+
+ proc.process(Collections.singletonList(watermarkMsg(REGION_R1, 1, 1000)));
+ Assert.assertEquals(1000, proc.getWatermark());
+ }
+}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java
index e5753bf1bd184..7f20f8cbfd03a 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java
@@ -79,6 +79,8 @@ public enum CnToDnAsyncRequestType {
TOPIC_PUSH_MULTI_META,
CONSUMER_GROUP_PUSH_ALL_META,
CONSUMER_GROUP_PUSH_SINGLE_META,
+ PULL_COMMIT_PROGRESS,
+ SUBSCRIPTION_PUSH_RUNTIME,
// TEMPLATE
UPDATE_TEMPLATE,
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java
index cd69f8b2c846d..4faea49d2fb7f 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java
@@ -47,6 +47,7 @@
import org.apache.iotdb.confignode.client.async.handlers.rpc.TreeDeviceViewFieldDetectionHandler;
import org.apache.iotdb.confignode.client.async.handlers.rpc.subscription.CheckSchemaRegionUsingTemplateRPCHandler;
import org.apache.iotdb.confignode.client.async.handlers.rpc.subscription.ConsumerGroupPushMetaRPCHandler;
+import org.apache.iotdb.confignode.client.async.handlers.rpc.subscription.PullCommitProgressRPCHandler;
import org.apache.iotdb.confignode.client.async.handlers.rpc.subscription.TopicPushMetaRPCHandler;
import org.apache.iotdb.mpp.rpc.thrift.TActiveTriggerInstanceReq;
import org.apache.iotdb.mpp.rpc.thrift.TAlterEncodingCompressorReq;
@@ -83,6 +84,7 @@
import org.apache.iotdb.mpp.rpc.thrift.TKillQueryInstanceReq;
import org.apache.iotdb.mpp.rpc.thrift.TNotifyRegionMigrationReq;
import org.apache.iotdb.mpp.rpc.thrift.TPipeHeartbeatReq;
+import org.apache.iotdb.mpp.rpc.thrift.TPullCommitProgressReq;
import org.apache.iotdb.mpp.rpc.thrift.TPushConsumerGroupMetaReq;
import org.apache.iotdb.mpp.rpc.thrift.TPushMultiPipeMetaReq;
import org.apache.iotdb.mpp.rpc.thrift.TPushMultiTopicMetaReq;
@@ -90,6 +92,7 @@
import org.apache.iotdb.mpp.rpc.thrift.TPushSingleConsumerGroupMetaReq;
import org.apache.iotdb.mpp.rpc.thrift.TPushSinglePipeMetaReq;
import org.apache.iotdb.mpp.rpc.thrift.TPushSingleTopicMetaReq;
+import org.apache.iotdb.mpp.rpc.thrift.TPushSubscriptionRuntimeReq;
import org.apache.iotdb.mpp.rpc.thrift.TPushTopicMetaReq;
import org.apache.iotdb.mpp.rpc.thrift.TRegionLeaderChangeReq;
import org.apache.iotdb.mpp.rpc.thrift.TRegionRouteReq;
@@ -224,6 +227,16 @@ protected void initActionMapBuilder() {
(req, client, handler) ->
client.pushSingleConsumerGroupMeta(
(TPushSingleConsumerGroupMetaReq) req, (ConsumerGroupPushMetaRPCHandler) handler));
+ actionMapBuilder.put(
+ CnToDnAsyncRequestType.PULL_COMMIT_PROGRESS,
+ (req, client, handler) ->
+ client.pullCommitProgress(
+ (TPullCommitProgressReq) req, (PullCommitProgressRPCHandler) handler));
+ actionMapBuilder.put(
+ CnToDnAsyncRequestType.SUBSCRIPTION_PUSH_RUNTIME,
+ (req, client, handler) ->
+ client.pushSubscriptionRuntime(
+ (TPushSubscriptionRuntimeReq) req, (DataNodeTSStatusRPCHandler) handler));
actionMapBuilder.put(
CnToDnAsyncRequestType.PIPE_HEARTBEAT,
(req, client, handler) ->
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java
index b2e2ec3232781..084998aa04825 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java
@@ -29,12 +29,14 @@
import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType;
import org.apache.iotdb.confignode.client.async.handlers.rpc.subscription.CheckSchemaRegionUsingTemplateRPCHandler;
import org.apache.iotdb.confignode.client.async.handlers.rpc.subscription.ConsumerGroupPushMetaRPCHandler;
+import org.apache.iotdb.confignode.client.async.handlers.rpc.subscription.PullCommitProgressRPCHandler;
import org.apache.iotdb.confignode.client.async.handlers.rpc.subscription.TopicPushMetaRPCHandler;
import org.apache.iotdb.mpp.rpc.thrift.TCheckSchemaRegionUsingTemplateResp;
import org.apache.iotdb.mpp.rpc.thrift.TCheckTimeSeriesExistenceResp;
import org.apache.iotdb.mpp.rpc.thrift.TCountPathsUsingTemplateResp;
import org.apache.iotdb.mpp.rpc.thrift.TDeviceViewResp;
import org.apache.iotdb.mpp.rpc.thrift.TFetchSchemaBlackListResp;
+import org.apache.iotdb.mpp.rpc.thrift.TPullCommitProgressResp;
import org.apache.iotdb.mpp.rpc.thrift.TPushConsumerGroupMetaResp;
import org.apache.iotdb.mpp.rpc.thrift.TPushPipeMetaResp;
import org.apache.iotdb.mpp.rpc.thrift.TPushTopicMetaResp;
@@ -169,6 +171,14 @@ public static DataNodeAsyncRequestRPCHandler buildHandler(
dataNodeLocationMap,
(Map) responseMap,
countDownLatch);
+ case PULL_COMMIT_PROGRESS:
+ return new PullCommitProgressRPCHandler(
+ requestType,
+ requestId,
+ targetDataNode,
+ dataNodeLocationMap,
+ (Map) responseMap,
+ countDownLatch);
case CHANGE_REGION_LEADER:
return new TransferLeaderRPCHandler(
requestType,
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeTSStatusRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeTSStatusRPCHandler.java
index 7c93f363dd4b8..bd8042071480a 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeTSStatusRPCHandler.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeTSStatusRPCHandler.java
@@ -48,22 +48,19 @@ public DataNodeTSStatusRPCHandler(
@Override
public void onComplete(TSStatus response) {
- // Put response
responseMap.put(requestId, response);
if (response.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- // Remove only if success
nodeLocationMap.remove(requestId);
LOGGER.info("Successfully {} on DataNode: {}", requestType, formattedTargetLocation);
} else {
- LOGGER.error(
+ logFailure(
"Failed to {} on DataNode: {}, response: {}",
requestType,
formattedTargetLocation,
response);
}
- // Always CountDown
countDownLatch.countDown();
}
@@ -76,14 +73,21 @@ public void onError(Exception e) {
+ formattedTargetLocation
+ ", exception: "
+ e.getMessage();
- LOGGER.error(errorMsg);
+ logFailure(errorMsg);
responseMap.put(
requestId,
new TSStatus(
RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode(), errorMsg)));
- // Always CountDown
countDownLatch.countDown();
}
+
+ private void logFailure(final String format, final Object... args) {
+ if (requestType == CnToDnAsyncRequestType.SUBSCRIPTION_PUSH_RUNTIME) {
+ LOGGER.warn(format, args);
+ } else {
+ LOGGER.error(format, args);
+ }
+ }
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/subscription/ConsumerGroupPushMetaRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/subscription/ConsumerGroupPushMetaRPCHandler.java
index 2938d4f85b7cd..67ee9f372d747 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/subscription/ConsumerGroupPushMetaRPCHandler.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/subscription/ConsumerGroupPushMetaRPCHandler.java
@@ -49,23 +49,19 @@ public ConsumerGroupPushMetaRPCHandler(
@Override
public void onComplete(TPushConsumerGroupMetaResp response) {
- // Put response
responseMap.put(requestId, response);
if (response.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- LOGGER.info("Successfully {} on DataNode: {}", requestType, formattedTargetLocation);
+ LOGGER.debug("Successfully {} on DataNode: {}", requestType, formattedTargetLocation);
} else {
- LOGGER.error(
+ LOGGER.warn(
"Failed to {} on DataNode: {}, response: {}",
requestType,
formattedTargetLocation,
response);
}
- // Always remove to avoid retrying
nodeLocationMap.remove(requestId);
-
- // Always CountDown
countDownLatch.countDown();
}
@@ -78,14 +74,13 @@ public void onError(Exception e) {
+ formattedTargetLocation
+ ", exception: "
+ e.getMessage();
- LOGGER.error(errorMsg, e);
+ LOGGER.warn(errorMsg);
responseMap.put(
requestId,
new TPushConsumerGroupMetaResp(
RpcUtils.getStatus(TSStatusCode.CONSUMER_PUSH_META_ERROR, errorMsg)));
- // Always CountDown
countDownLatch.countDown();
}
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/subscription/PullCommitProgressRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/subscription/PullCommitProgressRPCHandler.java
new file mode 100644
index 0000000000000..a34dd627f320f
--- /dev/null
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/subscription/PullCommitProgressRPCHandler.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.client.async.handlers.rpc.subscription;
+
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType;
+import org.apache.iotdb.confignode.client.async.handlers.rpc.DataNodeAsyncRequestRPCHandler;
+import org.apache.iotdb.mpp.rpc.thrift.TPullCommitProgressResp;
+import org.apache.iotdb.rpc.RpcUtils;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+
+public class PullCommitProgressRPCHandler
+ extends DataNodeAsyncRequestRPCHandler {
+ private static final Logger LOGGER = LoggerFactory.getLogger(PullCommitProgressRPCHandler.class);
+
+ public PullCommitProgressRPCHandler(
+ CnToDnAsyncRequestType requestType,
+ int requestId,
+ TDataNodeLocation targetDataNode,
+ Map dataNodeLocationMap,
+ Map responseMap,
+ CountDownLatch countDownLatch) {
+ super(requestType, requestId, targetDataNode, dataNodeLocationMap, responseMap, countDownLatch);
+ }
+
+ @Override
+ public void onComplete(TPullCommitProgressResp response) {
+ responseMap.put(requestId, response);
+
+ if (response.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ logSuspiciousRegionProgressPayloads(response);
+ LOGGER.info("Successfully {} on DataNode: {}", requestType, formattedTargetLocation);
+ } else {
+ LOGGER.error(
+ "Failed to {} on DataNode: {}, response: {}",
+ requestType,
+ formattedTargetLocation,
+ response);
+ }
+
+ nodeLocationMap.remove(requestId);
+ countDownLatch.countDown();
+ }
+
+ @Override
+ public void onError(Exception e) {
+ String errorMsg =
+ "Failed to "
+ + requestType
+ + " on DataNode: "
+ + formattedTargetLocation
+ + ", exception: "
+ + e.getMessage();
+ LOGGER.error(errorMsg, e);
+
+ responseMap.put(
+ requestId,
+ new TPullCommitProgressResp(
+ RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, errorMsg)));
+
+ countDownLatch.countDown();
+ }
+
+ private void logSuspiciousRegionProgressPayloads(final TPullCommitProgressResp response) {
+ if (response == null || !response.isSetCommitRegionProgress()) {
+ return;
+ }
+ for (final Map.Entry entry :
+ response.getCommitRegionProgress().entrySet()) {
+ if (isSuspiciousRegionProgressPayload(entry.getValue())) {
+ LOGGER.warn(
+ "PULL_COMMIT_PROGRESS confignode recv suspicious payload from DataNode {}, key={}, summary={}",
+ formattedTargetLocation,
+ entry.getKey(),
+ summarizeRegionProgressPayload(entry.getValue()));
+ }
+ }
+ }
+
+ private boolean isSuspiciousRegionProgressPayload(final java.nio.ByteBuffer buffer) {
+ if (buffer == null) {
+ return true;
+ }
+ final java.nio.ByteBuffer duplicate = buffer.slice();
+ if (duplicate.remaining() < Integer.BYTES) {
+ return true;
+ }
+ final int firstInt = duplicate.getInt();
+ return firstInt < 0 || firstInt > 1_000_000;
+ }
+
+ private String summarizeRegionProgressPayload(final java.nio.ByteBuffer buffer) {
+ if (buffer == null) {
+ return "null";
+ }
+ final int position = buffer.position();
+ final int limit = buffer.limit();
+ final int capacity = buffer.capacity();
+ final java.nio.ByteBuffer duplicate = buffer.slice();
+ final int remaining = duplicate.remaining();
+ final String firstIntSummary;
+ if (remaining >= Integer.BYTES) {
+ final int firstInt = duplicate.getInt();
+ firstIntSummary = firstInt + "(0x" + String.format("%08x", firstInt) + ")";
+ duplicate.position(0);
+ } else {
+ firstIntSummary = "n/a";
+ }
+ final int sampleLength = Math.min(16, remaining);
+ final byte[] sample = new byte[sampleLength];
+ duplicate.get(sample, 0, sampleLength);
+ return "pos="
+ + position
+ + ", limit="
+ + limit
+ + ", capacity="
+ + capacity
+ + ", remaining="
+ + remaining
+ + ", firstInt="
+ + firstIntSummary
+ + ", firstBytes="
+ + bytesToHex(sample);
+ }
+
+ private String bytesToHex(final byte[] bytes) {
+ if (bytes == null || bytes.length == 0) {
+ return "";
+ }
+ final StringBuilder builder = new StringBuilder(bytes.length * 2);
+ for (final byte b : bytes) {
+ builder.append(String.format("%02x", b));
+ }
+ return builder.toString();
+ }
+}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/subscription/TopicPushMetaRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/subscription/TopicPushMetaRPCHandler.java
index 91ffdd7232b3f..2f5e609f0cfec 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/subscription/TopicPushMetaRPCHandler.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/subscription/TopicPushMetaRPCHandler.java
@@ -48,23 +48,19 @@ public TopicPushMetaRPCHandler(
@Override
public void onComplete(TPushTopicMetaResp response) {
- // Put response
responseMap.put(requestId, response);
if (response.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- LOGGER.info("Successfully {} on DataNode: {}", requestType, formattedTargetLocation);
+ LOGGER.debug("Successfully {} on DataNode: {}", requestType, formattedTargetLocation);
} else {
- LOGGER.error(
+ LOGGER.warn(
"Failed to {} on DataNode: {}, response: {}",
requestType,
formattedTargetLocation,
response);
}
- // Always remove to avoid retrying
nodeLocationMap.remove(requestId);
-
- // Always CountDown
countDownLatch.countDown();
}
@@ -77,13 +73,12 @@ public void onError(Exception e) {
+ formattedTargetLocation
+ ", exception: "
+ e.getMessage();
- LOGGER.error(errorMsg, e);
+ LOGGER.warn(errorMsg);
responseMap.put(
requestId,
new TPushTopicMetaResp(RpcUtils.getStatus(TSStatusCode.TOPIC_PUSH_META_ERROR, errorMsg)));
- // Always CountDown
countDownLatch.countDown();
}
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java
index 7fd7cd029119a..662e5d4d445cb 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java
@@ -87,6 +87,7 @@
import org.apache.iotdb.confignode.consensus.request.write.region.PollRegionMaintainTaskPlan;
import org.apache.iotdb.confignode.consensus.request.write.region.PollSpecificRegionMaintainTaskPlan;
import org.apache.iotdb.confignode.consensus.request.write.subscription.consumer.AlterConsumerGroupPlan;
+import org.apache.iotdb.confignode.consensus.request.write.subscription.consumer.runtime.CommitProgressHandleMetaChangePlan;
import org.apache.iotdb.confignode.consensus.request.write.subscription.consumer.runtime.ConsumerGroupHandleMetaChangePlan;
import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.AlterMultipleTopicsPlan;
import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.AlterTopicPlan;
@@ -538,6 +539,9 @@ public static ConfigPhysicalPlan create(final ByteBuffer buffer) throws IOExcept
case ConsumerGroupHandleMetaChange:
plan = new ConsumerGroupHandleMetaChangePlan();
break;
+ case CommitProgressHandleMetaChange:
+ plan = new CommitProgressHandleMetaChangePlan();
+ break;
case PipeUnsetTemplate:
plan = new PipeUnsetSchemaTemplatePlan();
break;
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java
index 371435c9175bb..872ef0596d3c6 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java
@@ -323,6 +323,8 @@ public enum ConfigPhysicalPlanType {
ShowSubscription((short) 2000),
+ CommitProgressHandleMetaChange((short) 2001),
+
// Authority version after and equal 2.0
DropUserV2((short) 2100),
UpdateUserV2((short) 2101),
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/subscription/consumer/runtime/CommitProgressHandleMetaChangePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/subscription/consumer/runtime/CommitProgressHandleMetaChangePlan.java
new file mode 100644
index 0000000000000..387b0a43b4a61
--- /dev/null
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/subscription/consumer/runtime/CommitProgressHandleMetaChangePlan.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.consensus.request.write.subscription.consumer.runtime;
+
+import org.apache.iotdb.commons.subscription.meta.consumer.CommitProgressKeeper;
+import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan;
+import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/** Consensus plan for handling per-region commit progress meta changes. */
+public class CommitProgressHandleMetaChangePlan extends ConfigPhysicalPlan {
+
+ private Map regionProgressMap = new HashMap<>();
+
+ public CommitProgressHandleMetaChangePlan() {
+ super(ConfigPhysicalPlanType.CommitProgressHandleMetaChange);
+ }
+
+ public CommitProgressHandleMetaChangePlan(final Map regionProgressMap) {
+ super(ConfigPhysicalPlanType.CommitProgressHandleMetaChange);
+ this.regionProgressMap = regionProgressMap;
+ }
+
+ public Map getRegionProgressMap() {
+ return regionProgressMap;
+ }
+
+ @Override
+ protected void serializeImpl(final DataOutputStream stream) throws IOException {
+ stream.writeShort(getType().getPlanType());
+ stream.writeInt(regionProgressMap.size());
+ for (final Map.Entry entry : regionProgressMap.entrySet()) {
+ final byte[] keyBytes = entry.getKey().getBytes("UTF-8");
+ final ByteBuffer valueBuffer = entry.getValue().asReadOnlyBuffer();
+ valueBuffer.rewind();
+ final byte[] valueBytes = new byte[valueBuffer.remaining()];
+ valueBuffer.get(valueBytes);
+ stream.writeInt(keyBytes.length);
+ stream.write(keyBytes);
+ stream.writeInt(valueBytes.length);
+ stream.write(valueBytes);
+ }
+ }
+
+ @Override
+ protected void deserializeImpl(final ByteBuffer buffer) throws IOException {
+ regionProgressMap = CommitProgressKeeper.deserializeRegionProgressFromBuffer(buffer);
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ final CommitProgressHandleMetaChangePlan that = (CommitProgressHandleMetaChangePlan) obj;
+ return Objects.equals(this.regionProgressMap, that.regionProgressMap);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(regionProgressMap);
+ }
+}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
index f455edb26b8b1..41eaaf7440180 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
@@ -191,6 +191,8 @@
import org.apache.iotdb.confignode.rpc.thrift.TGetAllSubscriptionInfoResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetAllTemplatesResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetAllTopicInfoResp;
+import org.apache.iotdb.confignode.rpc.thrift.TGetCommitProgressReq;
+import org.apache.iotdb.confignode.rpc.thrift.TGetCommitProgressResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetDataNodeLocationsResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq;
import org.apache.iotdb.confignode.rpc.thrift.TGetJarInListReq;
@@ -254,6 +256,9 @@
import org.apache.iotdb.db.schemaengine.template.alter.TemplateAlterOperationUtil;
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.rpc.subscription.payload.poll.RegionProgress;
+import org.apache.iotdb.rpc.subscription.payload.poll.WriterId;
+import org.apache.iotdb.rpc.subscription.payload.poll.WriterProgress;
import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp;
@@ -264,6 +269,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.net.URL;
@@ -276,8 +283,10 @@
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -2508,6 +2517,83 @@ public TGetAllSubscriptionInfoResp getAllSubscriptionInfo() {
: new TGetAllSubscriptionInfoResp(status, Collections.emptyList());
}
+ public TGetCommitProgressResp getCommitProgress(TGetCommitProgressReq req) {
+ TSStatus status = confirmLeader();
+ if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ return new TGetCommitProgressResp(status);
+ }
+ final String key =
+ req.getConsumerGroupId()
+ + "##"
+ + req.getTopicName()
+ + "##"
+ + req.getRegionId()
+ + "##"
+ + req.getDataNodeId();
+ final String keyPrefix =
+ req.getConsumerGroupId() + "##" + req.getTopicName() + "##" + req.getRegionId() + "##";
+ final org.apache.iotdb.commons.subscription.meta.consumer.CommitProgressKeeper keeper =
+ subscriptionManager
+ .getSubscriptionCoordinator()
+ .getSubscriptionInfo()
+ .getCommitProgressKeeper();
+ final Map mergedWriterPositions = new LinkedHashMap<>();
+
+ for (final Map.Entry entry : keeper.getAllRegionProgress().entrySet()) {
+ if (!entry.getKey().startsWith(keyPrefix)) {
+ continue;
+ }
+ final RegionProgress regionProgress = deserializeRegionProgress(entry.getValue());
+ if (Objects.isNull(regionProgress)) {
+ continue;
+ }
+ for (final Map.Entry writerEntry :
+ regionProgress.getWriterPositions().entrySet()) {
+ mergedWriterPositions.merge(
+ writerEntry.getKey(),
+ writerEntry.getValue(),
+ (oldProgress, newProgress) ->
+ compareWriterProgress(newProgress, oldProgress) > 0 ? newProgress : oldProgress);
+ }
+ }
+ final TGetCommitProgressResp resp =
+ new TGetCommitProgressResp(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
+ if (!mergedWriterPositions.isEmpty()) {
+ resp.setCommittedRegionProgress(
+ serializeRegionProgress(new RegionProgress(mergedWriterPositions)));
+ }
+ return resp;
+ }
+
+ private static RegionProgress deserializeRegionProgress(final ByteBuffer buffer) {
+ if (Objects.isNull(buffer)) {
+ return null;
+ }
+ final ByteBuffer duplicate = buffer.asReadOnlyBuffer();
+ duplicate.rewind();
+ return RegionProgress.deserialize(duplicate);
+ }
+
+ private static ByteBuffer serializeRegionProgress(final RegionProgress regionProgress) {
+ try (final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ final DataOutputStream dos = new DataOutputStream(baos)) {
+ regionProgress.serialize(dos);
+ dos.flush();
+ return ByteBuffer.wrap(baos.toByteArray()).asReadOnlyBuffer();
+ } catch (final IOException e) {
+ throw new RuntimeException("Failed to serialize region progress " + regionProgress, e);
+ }
+ }
+
+ private static int compareWriterProgress(
+ final WriterProgress leftProgress, final WriterProgress rightProgress) {
+ int cmp = Long.compare(leftProgress.getPhysicalTime(), rightProgress.getPhysicalTime());
+ if (cmp != 0) {
+ return cmp;
+ }
+ return Long.compare(leftProgress.getLocalSeq(), rightProgress.getLocalSeq());
+ }
+
@Override
public TPipeConfigTransferResp handleTransferConfigPlan(TPipeConfigTransferReq req) {
TSStatus status = confirmLeader();
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
index 499c2fe30d3f0..cdc59261060f5 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
@@ -114,7 +114,9 @@
import org.apache.iotdb.confignode.procedure.impl.schema.table.view.SetViewPropertiesProcedure;
import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.CreateConsumerProcedure;
import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.DropConsumerProcedure;
+import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.runtime.CommitProgressSyncProcedure;
import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.runtime.ConsumerGroupMetaSyncProcedure;
+import org.apache.iotdb.confignode.procedure.impl.subscription.runtime.SubscriptionHandleLeaderChangeProcedure;
import org.apache.iotdb.confignode.procedure.impl.subscription.subscription.CreateSubscriptionProcedure;
import org.apache.iotdb.confignode.procedure.impl.subscription.subscription.DropSubscriptionProcedure;
import org.apache.iotdb.confignode.procedure.impl.subscription.topic.CreateTopicProcedure;
@@ -1665,6 +1667,21 @@ public void pipeHandleLeaderChange(
}
}
+ public void subscriptionHandleLeaderChange(
+ Map> regionGroupToOldAndNewLeaderPairMap,
+ long runtimeVersion) {
+ try {
+ final long procedureId =
+ executor.submitProcedure(
+ new SubscriptionHandleLeaderChangeProcedure(
+ regionGroupToOldAndNewLeaderPairMap, runtimeVersion));
+ LOGGER.info(
+ "SubscriptionHandleLeaderChangeProcedure was submitted, procedureId: {}.", procedureId);
+ } catch (Exception e) {
+ LOGGER.warn("SubscriptionHandleLeaderChangeProcedure was failed to submit.", e);
+ }
+ }
+
public void pipeHandleMetaChange(
boolean needWriteConsensusOnConfigNodes, boolean needPushPipeMetaToDataNodes) {
try {
@@ -1814,6 +1831,23 @@ public TSStatus consumerGroupMetaSync() {
}
}
+ public TSStatus commitProgressSync() {
+ try {
+ CommitProgressSyncProcedure procedure = new CommitProgressSyncProcedure();
+ executor.submitProcedure(procedure);
+ TSStatus status = waitingProcedureFinished(procedure);
+ if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ return status;
+ } else {
+ return new TSStatus(TSStatusCode.CONSUMER_PUSH_META_ERROR.getStatusCode())
+ .setMessage(wrapTimeoutMessageForPipeProcedure(status.getMessage()));
+ }
+ } catch (Exception e) {
+ return new TSStatus(TSStatusCode.CONSUMER_PUSH_META_ERROR.getStatusCode())
+ .setMessage(e.getMessage());
+ }
+ }
+
public TSStatus createSubscription(TSubscribeReq req) {
try {
CreateSubscriptionProcedure procedure = new CreateSubscriptionProcedure(req);
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java
index 993bfc0e40066..55d9417f30a2b 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java
@@ -88,6 +88,8 @@ public LoadManager(IManager configManager) {
this.topologyService = new TopologyService(configManager, loadCache::updateTopology);
this.eventService = new EventService(loadCache);
this.eventService.register(configManager.getPipeManager().getPipeRuntimeCoordinator());
+ this.eventService.register(
+ configManager.getSubscriptionManager().getSubscriptionLeaderChangeHandler());
this.eventService.register(routeBalancer);
this.eventService.register(topologyService);
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/SubscriptionManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/SubscriptionManager.java
index 1080b067fae82..ff06e20cf2dc7 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/SubscriptionManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/SubscriptionManager.java
@@ -20,17 +20,32 @@
package org.apache.iotdb.confignode.manager.subscription;
import org.apache.iotdb.confignode.manager.ConfigManager;
+import org.apache.iotdb.confignode.manager.subscription.runtime.SubscriptionLeaderChangeHandler;
+import org.apache.iotdb.confignode.manager.subscription.runtime.SubscriptionRuntimeCoordinator;
import org.apache.iotdb.confignode.persistence.subscription.SubscriptionInfo;
public class SubscriptionManager {
private final SubscriptionCoordinator subscriptionCoordinator;
+ private final SubscriptionRuntimeCoordinator subscriptionRuntimeCoordinator;
+ private final SubscriptionLeaderChangeHandler subscriptionLeaderChangeHandler;
public SubscriptionManager(ConfigManager configManager, SubscriptionInfo subscriptionInfo) {
this.subscriptionCoordinator = new SubscriptionCoordinator(configManager, subscriptionInfo);
+ this.subscriptionRuntimeCoordinator = new SubscriptionRuntimeCoordinator(configManager);
+ this.subscriptionLeaderChangeHandler =
+ new SubscriptionLeaderChangeHandler(subscriptionRuntimeCoordinator);
}
public SubscriptionCoordinator getSubscriptionCoordinator() {
return subscriptionCoordinator;
}
+
+ public SubscriptionRuntimeCoordinator getSubscriptionRuntimeCoordinator() {
+ return subscriptionRuntimeCoordinator;
+ }
+
+ public SubscriptionLeaderChangeHandler getSubscriptionLeaderChangeHandler() {
+ return subscriptionLeaderChangeHandler;
+ }
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/SubscriptionMetaSyncer.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/SubscriptionMetaSyncer.java
index de49987e13fbe..4931a2948fc61 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/SubscriptionMetaSyncer.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/SubscriptionMetaSyncer.java
@@ -106,6 +106,13 @@ private synchronized void sync() {
return;
}
+ // sync commit progress if syncing consumer group meta successfully
+ final TSStatus commitProgressSyncStatus = procedureManager.commitProgressSync();
+ if (commitProgressSyncStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ LOGGER.warn("Failed to sync commit progress. Result status: {}.", commitProgressSyncStatus);
+ return;
+ }
+
LOGGER.info(
"After this successful sync, if SubscriptionInfo is empty during this sync and has not been modified afterwards, all subsequent syncs will be skipped");
isLastSubscriptionSyncSuccessful = true;
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionLeaderChangeHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionLeaderChangeHandler.java
new file mode 100644
index 0000000000000..58cae4c8c2173
--- /dev/null
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionLeaderChangeHandler.java
@@ -0,0 +1,24 @@
+package org.apache.iotdb.confignode.manager.subscription.runtime;
+
+import org.apache.iotdb.confignode.manager.load.subscriber.ConsensusGroupStatisticsChangeEvent;
+import org.apache.iotdb.confignode.manager.load.subscriber.IClusterStatusSubscriber;
+import org.apache.iotdb.confignode.manager.load.subscriber.NodeStatisticsChangeEvent;
+
+public class SubscriptionLeaderChangeHandler implements IClusterStatusSubscriber {
+
+ private final SubscriptionRuntimeCoordinator runtimeCoordinator;
+
+ public SubscriptionLeaderChangeHandler(final SubscriptionRuntimeCoordinator runtimeCoordinator) {
+ this.runtimeCoordinator = runtimeCoordinator;
+ }
+
+ @Override
+ public void onNodeStatisticsChanged(final NodeStatisticsChangeEvent event) {
+ runtimeCoordinator.handleNodeStatisticsChange(event);
+ }
+
+ @Override
+ public void onConsensusGroupStatisticsChanged(final ConsensusGroupStatisticsChangeEvent event) {
+ runtimeCoordinator.handleLeaderChangeEvent(event);
+ }
+}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionRuntimeCoordinator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionRuntimeCoordinator.java
new file mode 100644
index 0000000000000..ec3799118fc0e
--- /dev/null
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionRuntimeCoordinator.java
@@ -0,0 +1,149 @@
+package org.apache.iotdb.confignode.manager.subscription.runtime;
+
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta;
+import org.apache.iotdb.confignode.manager.ConfigManager;
+import org.apache.iotdb.confignode.manager.load.cache.node.NodeStatistics;
+import org.apache.iotdb.confignode.manager.load.subscriber.ConsensusGroupStatisticsChangeEvent;
+import org.apache.iotdb.confignode.manager.load.subscriber.NodeStatisticsChangeEvent;
+import org.apache.iotdb.rpc.subscription.config.TopicConstant;
+
+import org.apache.tsfile.utils.Pair;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class SubscriptionRuntimeCoordinator {
+
+ private final ConfigManager configManager;
+ private final Map> regionGroupToRuntimeLeaderPairMap =
+ new HashMap<>();
+ private final AtomicLong runtimeVersionGenerator = new AtomicLong(System.currentTimeMillis());
+
+ public SubscriptionRuntimeCoordinator(final ConfigManager configManager) {
+ this.configManager = configManager;
+ }
+
+ public synchronized void handleLeaderChangeEvent(
+ final ConsensusGroupStatisticsChangeEvent event) {
+ if (!hasAnyConsensusBasedTopic()) {
+ return;
+ }
+
+ final Map> refreshMap = new HashMap<>();
+ event
+ .getDifferentConsensusGroupStatisticsMap()
+ .forEach(
+ (regionGroupId, pair) -> {
+ if (regionGroupId.getType() != TConsensusGroupType.DataRegion) {
+ return;
+ }
+ final int oldLeaderNodeId = pair.left == null ? -1 : pair.left.getLeaderId();
+ final int newLeaderNodeId = pair.right == null ? -1 : pair.right.getLeaderId();
+ if (oldLeaderNodeId == newLeaderNodeId) {
+ return;
+ }
+ updateRuntimeLeaderPair(regionGroupId, oldLeaderNodeId, newLeaderNodeId, refreshMap);
+ });
+
+ submitRuntimeRefresh(refreshMap);
+ }
+
+ public synchronized void handleNodeStatisticsChange(final NodeStatisticsChangeEvent event) {
+ if (!hasAnyConsensusBasedTopic()) {
+ return;
+ }
+
+ final boolean shouldRefreshRuntime =
+ event.getDifferentNodeStatisticsMap().values().stream()
+ .anyMatch(
+ pair -> {
+ final NodeStatus oldStatus = getNodeStatus(pair.getLeft());
+ final NodeStatus newStatus = getNodeStatus(pair.getRight());
+ return oldStatus != newStatus
+ && (isRuntimeSensitiveStatus(oldStatus)
+ || isRuntimeSensitiveStatus(newStatus));
+ });
+ if (!shouldRefreshRuntime) {
+ return;
+ }
+
+ seedRuntimeLeaderPairsFromCurrentLeaders();
+ submitRuntimeRefresh(new HashMap<>(regionGroupToRuntimeLeaderPairMap));
+ }
+
+ public boolean hasAnyConsensusBasedTopic() {
+ for (final TopicMeta topicMeta :
+ configManager
+ .getSubscriptionManager()
+ .getSubscriptionCoordinator()
+ .getSubscriptionInfo()
+ .getAllTopicMeta()) {
+ final String topicMode =
+ topicMeta
+ .getConfig()
+ .getStringOrDefault(TopicConstant.MODE_KEY, TopicConstant.MODE_DEFAULT_VALUE);
+ final String topicFormat =
+ topicMeta
+ .getConfig()
+ .getStringOrDefault(TopicConstant.FORMAT_KEY, TopicConstant.FORMAT_DEFAULT_VALUE);
+ if (TopicConstant.MODE_LIVE_VALUE.equalsIgnoreCase(topicMode)
+ && !TopicConstant.FORMAT_TS_FILE_HANDLER_VALUE.equalsIgnoreCase(topicFormat)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private void updateRuntimeLeaderPair(
+ final TConsensusGroupId regionGroupId,
+ final int oldLeaderNodeId,
+ final int newLeaderNodeId,
+ final Map> refreshMap) {
+ if (newLeaderNodeId < 0) {
+ regionGroupToRuntimeLeaderPairMap.remove(regionGroupId);
+ return;
+ }
+ final Pair runtimeLeaderPair = new Pair<>(oldLeaderNodeId, newLeaderNodeId);
+ regionGroupToRuntimeLeaderPairMap.put(regionGroupId, runtimeLeaderPair);
+ refreshMap.put(regionGroupId, runtimeLeaderPair);
+ }
+
+ private void seedRuntimeLeaderPairsFromCurrentLeaders() {
+ configManager
+ .getLoadManager()
+ .getRegionLeaderMap()
+ .forEach(
+ (regionGroupId, leaderId) -> {
+ if (regionGroupId.getType() == TConsensusGroupType.DataRegion && leaderId >= 0) {
+ regionGroupToRuntimeLeaderPairMap.putIfAbsent(
+ regionGroupId, new Pair<>(-1, leaderId));
+ }
+ });
+ }
+
+ private void submitRuntimeRefresh(
+ final Map> regionGroupToOldAndNewLeaderPairMap) {
+ if (regionGroupToOldAndNewLeaderPairMap.isEmpty()) {
+ return;
+ }
+ configManager
+ .getProcedureManager()
+ .subscriptionHandleLeaderChange(
+ regionGroupToOldAndNewLeaderPairMap,
+ runtimeVersionGenerator.updateAndGet(
+ currentRuntimeVersion ->
+ Math.max(currentRuntimeVersion + 1, System.currentTimeMillis())));
+ }
+
+ private static NodeStatus getNodeStatus(final NodeStatistics statistics) {
+ return statistics == null ? NodeStatus.Unknown : statistics.getStatus();
+ }
+
+ private static boolean isRuntimeSensitiveStatus(final NodeStatus status) {
+ return status == NodeStatus.Unknown || status == NodeStatus.Removing;
+ }
+}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
index 8016690d17c9a..1d232ec87a364 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
@@ -111,6 +111,7 @@
import org.apache.iotdb.confignode.consensus.request.write.region.OfferRegionMaintainTasksPlan;
import org.apache.iotdb.confignode.consensus.request.write.region.PollSpecificRegionMaintainTaskPlan;
import org.apache.iotdb.confignode.consensus.request.write.subscription.consumer.AlterConsumerGroupPlan;
+import org.apache.iotdb.confignode.consensus.request.write.subscription.consumer.runtime.CommitProgressHandleMetaChangePlan;
import org.apache.iotdb.confignode.consensus.request.write.subscription.consumer.runtime.ConsumerGroupHandleMetaChangePlan;
import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.AlterMultipleTopicsPlan;
import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.AlterTopicPlan;
@@ -636,6 +637,9 @@ public TSStatus executeNonQueryPlan(ConfigPhysicalPlan physicalPlan)
case ConsumerGroupHandleMetaChange:
return subscriptionInfo.handleConsumerGroupMetaChanges(
(ConsumerGroupHandleMetaChangePlan) physicalPlan);
+ case CommitProgressHandleMetaChange:
+ return subscriptionInfo.handleCommitProgressChanges(
+ (CommitProgressHandleMetaChangePlan) physicalPlan);
case AlterConsumerGroup:
return subscriptionInfo.alterConsumerGroup((AlterConsumerGroupPlan) physicalPlan);
case TopicHandleMetaChange:
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfo.java
index 0c262655156d3..2ce4805dc14f5 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfo.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfo.java
@@ -21,12 +21,14 @@
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.snapshot.SnapshotProcessor;
+import org.apache.iotdb.commons.subscription.meta.consumer.CommitProgressKeeper;
import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerGroupMeta;
import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerGroupMetaKeeper;
import org.apache.iotdb.commons.subscription.meta.subscription.SubscriptionMeta;
import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta;
import org.apache.iotdb.commons.subscription.meta.topic.TopicMetaKeeper;
import org.apache.iotdb.confignode.consensus.request.write.subscription.consumer.AlterConsumerGroupPlan;
+import org.apache.iotdb.confignode.consensus.request.write.subscription.consumer.runtime.CommitProgressHandleMetaChangePlan;
import org.apache.iotdb.confignode.consensus.request.write.subscription.consumer.runtime.ConsumerGroupHandleMetaChangePlan;
import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.AlterMultipleTopicsPlan;
import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.AlterTopicPlan;
@@ -42,6 +44,8 @@
import org.apache.iotdb.confignode.rpc.thrift.TUnsubscribeReq;
import org.apache.iotdb.consensus.common.DataSet;
import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.rpc.subscription.config.TopicConfig;
+import org.apache.iotdb.rpc.subscription.config.TopicConstant;
import org.apache.iotdb.rpc.subscription.exception.SubscriptionException;
import org.apache.thrift.annotation.Nullable;
@@ -54,7 +58,9 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
@@ -72,6 +78,7 @@ public class SubscriptionInfo implements SnapshotProcessor {
private final TopicMetaKeeper topicMetaKeeper;
private final ConsumerGroupMetaKeeper consumerGroupMetaKeeper;
+ private final CommitProgressKeeper commitProgressKeeper;
private final ReentrantReadWriteLock subscriptionInfoLock = new ReentrantReadWriteLock(true);
@@ -81,6 +88,7 @@ public class SubscriptionInfo implements SnapshotProcessor {
public SubscriptionInfo() {
this.topicMetaKeeper = new TopicMetaKeeper();
this.consumerGroupMetaKeeper = new ConsumerGroupMetaKeeper();
+ this.commitProgressKeeper = new CommitProgressKeeper();
this.subscriptionInfoVersion = new SubscriptionInfoVersion();
}
@@ -158,6 +166,8 @@ public boolean validateBeforeCreatingTopic(TCreateTopicReq createTopicReq)
private boolean checkBeforeCreateTopicInternal(TCreateTopicReq createTopicReq)
throws SubscriptionException {
+ validateTopicConfig(new TopicConfig(safeTopicAttributes(createTopicReq.getTopicAttributes())));
+
if (!isTopicExisted(createTopicReq.getTopicName())) {
return true;
}
@@ -247,6 +257,8 @@ public void validateBeforeAlteringTopic(TopicMeta topicMeta) throws Subscription
}
private void checkBeforeAlteringTopicInternal(TopicMeta topicMeta) throws SubscriptionException {
+ validateTopicConfig(topicMeta.getConfig());
+
if (isTopicExisted(topicMeta.getTopicName())) {
return;
}
@@ -258,6 +270,28 @@ private void checkBeforeAlteringTopicInternal(TopicMeta topicMeta) throws Subscr
throw new SubscriptionException(exceptionMessage);
}
+ private Map safeTopicAttributes(@Nullable final Map attributes) {
+ return Objects.nonNull(attributes) ? attributes : Collections.emptyMap();
+ }
+
+ private void validateTopicConfig(final TopicConfig topicConfig) throws SubscriptionException {
+ final String orderMode = topicConfig.getOrderMode();
+ if (TopicConfig.isValidOrderMode(orderMode)) {
+ return;
+ }
+
+ final String exceptionMessage =
+ String.format(
+ "Failed to create or alter topic, unsupported %s=%s, expected one of [%s, %s, %s]",
+ TopicConstant.ORDER_MODE_KEY,
+ orderMode,
+ TopicConstant.ORDER_MODE_LEADER_ONLY_VALUE,
+ TopicConstant.ORDER_MODE_MULTI_WRITER_VALUE,
+ TopicConstant.ORDER_MODE_PER_WRITER_VALUE);
+ LOGGER.warn(exceptionMessage);
+ throw new SubscriptionException(exceptionMessage);
+ }
+
public boolean isTopicExisted(String topicName) {
acquireReadLock();
try {
@@ -566,6 +600,21 @@ public TSStatus handleConsumerGroupMetaChanges(ConsumerGroupHandleMetaChangePlan
}
}
+ public TSStatus handleCommitProgressChanges(CommitProgressHandleMetaChangePlan plan) {
+ acquireWriteLock();
+ try {
+ LOGGER.info("Handling commit progress meta changes ...");
+ commitProgressKeeper.replaceAll(plan.getRegionProgressMap());
+ return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
+ } finally {
+ releaseWriteLock();
+ }
+ }
+
+ public CommitProgressKeeper getCommitProgressKeeper() {
+ return commitProgressKeeper;
+ }
+
///////////////////////////////// Subscription /////////////////////////////////
public void validateBeforeSubscribe(TSubscribeReq subscribeReq) throws SubscriptionException {
@@ -740,6 +789,7 @@ public boolean processTakeSnapshot(File snapshotDir) throws IOException {
try (final FileOutputStream fileOutputStream = new FileOutputStream(snapshotFile)) {
topicMetaKeeper.processTakeSnapshot(fileOutputStream);
consumerGroupMetaKeeper.processTakeSnapshot(fileOutputStream);
+ commitProgressKeeper.processTakeSnapshot(fileOutputStream);
fileOutputStream.getFD().sync();
}
@@ -764,6 +814,7 @@ public void processLoadSnapshot(File snapshotDir) throws IOException {
try (final FileInputStream fileInputStream = new FileInputStream(snapshotFile)) {
topicMetaKeeper.processLoadSnapshot(fileInputStream);
consumerGroupMetaKeeper.processLoadSnapshot(fileInputStream);
+ commitProgressKeeper.processLoadSnapshot(fileInputStream);
}
} finally {
releaseWriteLock();
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java
index 960d0a7977f51..d271d5ef33b9c 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java
@@ -70,6 +70,8 @@
import org.apache.iotdb.mpp.rpc.thrift.TInactiveTriggerInstanceReq;
import org.apache.iotdb.mpp.rpc.thrift.TInvalidateCacheReq;
import org.apache.iotdb.mpp.rpc.thrift.TNotifyRegionMigrationReq;
+import org.apache.iotdb.mpp.rpc.thrift.TPullCommitProgressReq;
+import org.apache.iotdb.mpp.rpc.thrift.TPullCommitProgressResp;
import org.apache.iotdb.mpp.rpc.thrift.TPushConsumerGroupMetaReq;
import org.apache.iotdb.mpp.rpc.thrift.TPushConsumerGroupMetaResp;
import org.apache.iotdb.mpp.rpc.thrift.TPushMultiPipeMetaReq;
@@ -79,12 +81,15 @@
import org.apache.iotdb.mpp.rpc.thrift.TPushSingleConsumerGroupMetaReq;
import org.apache.iotdb.mpp.rpc.thrift.TPushSinglePipeMetaReq;
import org.apache.iotdb.mpp.rpc.thrift.TPushSingleTopicMetaReq;
+import org.apache.iotdb.mpp.rpc.thrift.TPushSubscriptionRuntimeReq;
import org.apache.iotdb.mpp.rpc.thrift.TPushTopicMetaReq;
import org.apache.iotdb.mpp.rpc.thrift.TPushTopicMetaResp;
+import org.apache.iotdb.mpp.rpc.thrift.TSubscriptionRuntimeStateEntry;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.thrift.TException;
import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.utils.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -93,8 +98,10 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
@@ -848,6 +855,85 @@ public List dropSingleConsumerGroupOnDataNode(String consumerGroupName
.collect(Collectors.toList());
}
+ public Map pullCommitProgressFromDataNodes() {
+ final Map dataNodeLocationMap =
+ configManager.getNodeManager().getRegisteredDataNodeLocations();
+ final TPullCommitProgressReq request = new TPullCommitProgressReq();
+
+ final DataNodeAsyncRequestContext
+ clientHandler =
+ new DataNodeAsyncRequestContext<>(
+ CnToDnAsyncRequestType.PULL_COMMIT_PROGRESS, request, dataNodeLocationMap);
+ CnToDnInternalServiceAsyncRequestManager.getInstance()
+ .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
+ clientHandler,
+ PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
+ return clientHandler.getResponseMap();
+ }
+
+ public Map pushSubscriptionRuntimeStatesToDataNodes(
+ final Map> regionGroupToOldAndNewLeaderPairMap,
+ final long runtimeVersion) {
+ final Map dataNodeLocationMap =
+ configManager.getNodeManager().getRegisteredDataNodeLocations();
+ final Map dataRegionReplicaSetMap =
+ getPartitionManager().getAllReplicaSetsMap(TConsensusGroupType.DataRegion);
+ final Set