feat(iceberg): add external Iceberg connector

.github/workflows/iceberg-connector.yml

@@ -0,0 +1,72 @@
+name: Iceberg Connector E2E
+
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/iceberg-connector.yml'
+      - 'Makefile'
+      - 'optools/iceberg_ci.bash'
+      - 'optools/iceberg_external_runner.py'
+      - 'etc/launch-minio-local/**'
+      - 'test/iceberg/**'
+      - 'pkg/iceberg/**'
+      - 'pkg/sql/iceberg/**'
+      - 'pkg/sql/plan/**'
+      - 'pkg/sql/compile/**'
+      - 'pkg/frontend/**'
+      - 'pkg/sql/parsers/**'
+      - 'pkg/config/**'
+      - 'pkg/bootstrap/**'
+      - 'pkg/sql/colexec/external/**'
+      - 'pkg/sql/colexec/icebergdelete/**'
+      - 'pkg/sql/colexec/icebergwrite/**'
+      - 'pkg/embed/**'
+      - 'cmd/mo-service/**'
+      - 'pkg/cnservice/**'
+      - 'proto/pipeline.proto'
+      - 'pkg/pb/pipeline/**'
+
+permissions:
+  contents: read
+
+jobs:
+  iceberg-e2e-local:
+    name: Iceberg E2E Local
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    continue-on-error: true
+    env:
+      MO_ICEBERG_REPORT_DIR: test/iceberg/reports/iceberg-e2e-local-${{ github.run_id }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - name: Run local Nessie/MinIO/MO Iceberg E2E
+        run: make test-iceberg-e2e-local
+
+      - name: Summarize Iceberg E2E
+        if: always()
+        run: |
+          {
+            echo "## Iceberg E2E local"
+            echo ""
+            echo "Mode: advisory / non-blocking"
+            echo "Artifact: \`iceberg-e2e-local-${{ github.sha }}-${{ github.run_id }}\`"
+            echo ""
+            if [ -f "${MO_ICEBERG_REPORT_DIR}/summary.md" ]; then
+              cat "${MO_ICEBERG_REPORT_DIR}/summary.md"
+            else
+              echo "No summary was created."
+            fi
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: iceberg-e2e-local-${{ github.sha }}-${{ github.run_id }}
+          path: |
+            test/iceberg/reports/**

.gitignore

@@ -16,7 +16,10 @@
 *.swp
 *.fatbin
 *.py
+!pkg/iceberg/metadata/testdata/generate_golden_vectors.py
+!optools/iceberg_external_runner.py
 *.sh
+!etc/launch-minio-local/tier-b/seed-nyc-tlc-iceberg.sh
 bin/
 y.go
 *.output
@@ -30,6 +33,9 @@ tags
 coverage.txt
 out.txt
 logs/
+test/iceberg/reports/
+**/derby.log
+**/metastore_db/
 prometheus-local-data/
 prometheus-data/
 grafana-local-data/

.golangci.yml

@@ -28,6 +28,19 @@ linters:
           deny:
             - pkg: github.com/OpenPeeDeeP/depguard
               desc: example to deny some package
+        iceberg-adapter-boundary:
+          list-mode: lax
+          files:
+            - $all
+            - '!$test'
+            - '!pkg/iceberg/adapter/iceberggo/**/*.go'
+          deny:
+            - pkg: github.com/apache/iceberg-go
+              desc: Iceberg dependency must stay behind pkg/iceberg/adapter/iceberggo
+            - pkg: github.com/apache/arrow-go
+              desc: Arrow dependency must not leak outside the Iceberg adapter
+            - pkg: github.com/apache/arrow/go
+              desc: Arrow dependency must not leak outside the Iceberg adapter
     govet:
       disable:
         - fieldalignment

Makefile

@@ -106,6 +106,13 @@ help:
 	@echo "  make ut                 - Run unit tests"
 	@echo "  make ci                 - Run CI tests (BVT + optional UT)"
 	@echo "  make compose            - Run docker compose BVT tests"
+	@echo "  make test-iceberg-e2e-local - Run local Nessie/MinIO/MO Iceberg E2E smoke"
+	@echo "  make test-iceberg-local - Run legacy local Iceberg CI gates"
+	@echo "  make test-iceberg-nightly - Run enabled Iceberg external nightly gates"
+	@echo "  make test-iceberg-golden-real - Run real-file Iceberg golden cross-engine scenarios"
+	@echo "  make test-iceberg-external-templates - Validate external Iceberg scenario templates"
+	@echo "  make test-iceberg-readiness - Generate external Iceberg test readiness report"
+	@echo "  make test-iceberg-external-coverage - Verify external reports cover expected ICE-TEST ids"
 	@echo ""
 	@echo "Local Development with MinIO:"
 	@echo "  make dev-up-minio-local     - Start MinIO service (local storage)"
@@ -114,6 +121,14 @@ help:
 	@echo "  make dev-status-minio-local - Show MinIO service status"
 	@echo "  make dev-logs-minio-local   - Show MinIO logs"
 	@echo "  make dev-clean-minio-local  - Clean MinIO data"
+	@echo "  make dev-up-iceberg-tier-a  - Start MinIO + Nessie for Iceberg Tier A tests"
+	@echo "  make dev-up-iceberg-tier-a-brew - Start Iceberg Tier A services via local brew binaries"
+	@echo "  make dev-down-iceberg-tier-a-brew - Stop local brew Iceberg Tier A services"
+	@echo "  make dev-status-iceberg-tier-a-brew - Show local brew Iceberg Tier A service status"
+	@echo "  make dev-seed-iceberg-tier-a - Seed deterministic Iceberg Tier A tables"
+	@echo "  make dev-test-iceberg-tier-a - Run Iceberg Tier A integration tests"
+	@echo "  make dev-seed-iceberg-tier-b-nyc-tlc - Seed NYC TLC public dataset into local Iceberg"
+	@echo "  make dev-test-iceberg-tier-b-nyc-tlc - Run NYC TLC Tier B public dataset checks"
 	@echo "  make launch-minio           - Build and start MO with MinIO storage"
 	@echo "  make launch-minio-debug     - Build (debug) and start MO with MinIO"
 	@echo ""
@@ -308,6 +323,7 @@ ifeq ($(UNAME_S),darwin)
 else
 	@cd optools && timeout 60m ./run_ut.sh UT $(SKIP_TEST)
 endif
+	@optools/iceberg_ci.bash core
 
 ###############################################################################
 # bvt and unit test
@@ -334,6 +350,66 @@ ci-clean:
 	@docker rmi matrixorigin/matrixone:local-ci
 	@docker image prune -f
 
+.PHONY: test-iceberg-core
+test-iceberg-core:
+	@optools/iceberg_ci.bash core
+
+.PHONY: test-iceberg-embedded
+test-iceberg-embedded:
+	@optools/iceberg_ci.bash embedded
+
+.PHONY: test-iceberg-adapter
+test-iceberg-adapter:
+	@optools/iceberg_ci.bash adapter
+
+.PHONY: test-iceberg-golden
+test-iceberg-golden:
+	@optools/iceberg_ci.bash golden
+
+.PHONY: test-iceberg-golden-real
+test-iceberg-golden-real:
+	@optools/iceberg_ci.bash golden-real
+
+.PHONY: test-iceberg-external-templates
+test-iceberg-external-templates:
+	@optools/iceberg_ci.bash external-templates
+
+.PHONY: test-iceberg-coverage
+test-iceberg-coverage:
+	@optools/iceberg_ci.bash coverage
+
+.PHONY: test-iceberg-preflight
+test-iceberg-preflight:
+	@optools/iceberg_ci.bash preflight
+
+.PHONY: test-iceberg-artifact
+test-iceberg-artifact:
+	@optools/iceberg_ci.bash artifact
+
+.PHONY: test-iceberg-external-coverage
+test-iceberg-external-coverage:
+	@optools/iceberg_ci.bash external-coverage
+
+.PHONY: test-iceberg-dashboard
+test-iceberg-dashboard:
+	@optools/iceberg_ci.bash dashboard
+
+.PHONY: test-iceberg-readiness
+test-iceberg-readiness:
+	@optools/iceberg_ci.bash readiness
+
+.PHONY: test-iceberg-e2e-local
+test-iceberg-e2e-local:
+	@optools/iceberg_ci.bash e2e-local
+
+.PHONY: test-iceberg-local
+test-iceberg-local:
+	@optools/iceberg_ci.bash local
+
+.PHONY: test-iceberg-nightly
+test-iceberg-nightly:
+	@optools/iceberg_ci.bash nightly
+
 
 ###############################################################################
 # docker compose bvt test
@@ -962,6 +1038,7 @@ dev-up-minio-local:
 	@echo "✅ MinIO started!"
 	@echo "  - API: http://127.0.0.1:9000"
 	@echo "  - Console: http://127.0.0.1:9001"
+	@echo "  - Nessie: http://127.0.0.1:19120"
 	@echo "  - Access Key: minio"
 	@echo "  - Secret Key: minio123"
 	@echo "  - Data directory: $(MINIO_DATA_DIR)"
@@ -986,6 +1063,77 @@ dev-status-minio-local:
 dev-logs-minio-local:
 	@cd $(MINIO_DIR) && docker compose logs -f minio
 
+.PHONY: dev-up-iceberg-tier-a
+dev-up-iceberg-tier-a: dev-up-minio-local
+	@echo "Waiting for Nessie Iceberg REST catalog..."
+	@for i in $$(seq 1 60); do \
+		if curl -fsS --max-time 5 http://127.0.0.1:19120/iceberg/v1/config >/dev/null 2>&1; then \
+			break; \
+		fi; \
+		if [ "$$i" -eq 60 ]; then \
+			echo "Timed out waiting for Nessie Iceberg REST catalog" >&2; \
+			cd $(MINIO_DIR) && docker compose logs --tail=80 nessie >&2 || true; \
+			exit 1; \
+		fi; \
+		sleep 1; \
+	done
+	@echo "✅ Iceberg Tier A services started"
+	@echo "  - REST catalog: http://127.0.0.1:19120/iceberg"
+	@echo "  - Warehouse: s3://mo-iceberg/warehouse"
+
+.PHONY: dev-down-iceberg-tier-a
+dev-down-iceberg-tier-a: dev-down-minio-local
+
+.PHONY: dev-up-iceberg-tier-a-brew
+dev-up-iceberg-tier-a-brew:
+	@$(MINIO_DIR)/tier-a/start-brew-tier-a.sh
+
+.PHONY: dev-down-iceberg-tier-a-brew
+dev-down-iceberg-tier-a-brew:
+	@$(MINIO_DIR)/tier-a/stop-brew-tier-a.sh
+
+.PHONY: dev-status-iceberg-tier-a-brew
+dev-status-iceberg-tier-a-brew:
+	@printf "MinIO: "; curl -fsS http://127.0.0.1:9000/minio/health/live >/dev/null && echo up || echo down
+	@printf "Nessie: "; curl -fsS http://127.0.0.1:19120/iceberg/v1/config >/dev/null && echo up || echo down
+
+.PHONY: dev-logs-iceberg-tier-a-brew
+dev-logs-iceberg-tier-a-brew:
+	@tail -n 200 -f $(MINIO_DIR)/mo-data/logs/minio.log $(MINIO_DIR)/mo-data/logs/nessie.log
+
+.PHONY: dev-status-iceberg-tier-a
+dev-status-iceberg-tier-a:
+	@cd $(MINIO_DIR) && docker compose ps
+
+.PHONY: dev-logs-iceberg-tier-a
+dev-logs-iceberg-tier-a:
+	@cd $(MINIO_DIR) && docker compose logs -f minio nessie
+
+.PHONY: dev-seed-iceberg-tier-a
+dev-seed-iceberg-tier-a: dev-up-iceberg-tier-a
+	@$(MINIO_DIR)/tier-a/seed-iceberg-tier-a.sh
+
+.PHONY: dev-test-iceberg-tier-a
+dev-test-iceberg-tier-a:
+	@test -f $(MINIO_DIR)/tier-a/tier_a.generated.env || (echo "Missing $(MINIO_DIR)/tier-a/tier_a.generated.env. Run make dev-seed-iceberg-tier-a first."; exit 1)
+	@. $(MINIO_DIR)/tier-a/tier_a.generated.env && \
+		MO_ICEBERG_ALLOW_PLAIN_HTTP=1 \
+		MO_ICEBERG_REPORT_DIR=$${MO_ICEBERG_REPORT_DIR:-test/iceberg/reports/run_$$(date -u +%Y%m%dT%H%M%SZ)} \
+		go test ./pkg/sql/iceberg -run TestIcebergTierA -count=1
+
+.PHONY: dev-seed-iceberg-tier-b-nyc-tlc
+dev-seed-iceberg-tier-b-nyc-tlc: dev-up-iceberg-tier-a
+	@$(MINIO_DIR)/tier-b/seed-nyc-tlc-iceberg.sh
+
+.PHONY: dev-test-iceberg-tier-b-nyc-tlc
+dev-test-iceberg-tier-b-nyc-tlc:
+	@test -f $(MINIO_DIR)/tier-b/tier_b_nyc_tlc.generated.env || (echo "Missing $(MINIO_DIR)/tier-b/tier_b_nyc_tlc.generated.env. Run make dev-seed-iceberg-tier-b-nyc-tlc first."; exit 1)
+	@. $(MINIO_DIR)/tier-b/tier_b_nyc_tlc.generated.env && \
+		MO_ICEBERG_ALLOW_PLAIN_HTTP=1 \
+		MO_ICEBERG_CI_PROFILE=tier-b \
+		MO_ICEBERG_REPORT_DIR=$${MO_ICEBERG_REPORT_DIR:-test/iceberg/reports/nyc_tlc_$$(date -u +%Y%m%dT%H%M%SZ)} \
+		$(MAKE) test-iceberg-nightly
+
 .PHONY: dev-clean-minio-local
 dev-clean-minio-local:
 	@echo "WARNING: This will delete all MinIO data!"
@@ -1010,15 +1158,15 @@ launch-minio: build dev-up-minio-local
 	@echo "Starting MatrixOne with MinIO storage..."
 	@echo "  Launch config: $(MINIO_DIR)/launch.toml"
 	@echo ""
-	@./mo-service -launch $(MINIO_DIR)/launch.toml
+	@MO_ICEBERG_ALLOW_PLAIN_HTTP=1 ./mo-service -launch $(MINIO_DIR)/launch.toml
 
 .PHONY: launch-minio-debug
 launch-minio-debug: debug dev-up-minio-local
 	@echo ""
 	@echo "Starting MatrixOne (debug mode) with MinIO storage..."
 	@echo "  Launch config: $(MINIO_DIR)/launch.toml"
 	@echo ""
-	@./mo-service -launch $(MINIO_DIR)/launch.toml
+	@MO_ICEBERG_ALLOW_PLAIN_HTTP=1 ./mo-service -launch $(MINIO_DIR)/launch.toml
 
 ###############################################################################
 # clean

etc/launch-dynamic-cn/cn.toml.base

@@ -46,3 +46,10 @@ type = "distributed-tae"
 [cn.frontend]
 port = %d
 unix-socket = "/tmp/mysql%d.sock"
+
+[cn.frontend.iceberg]
+enable = true
+enable-write = true
+enable-delete = true
+enable-dml = true
+enable-maintenance = true

etc/launch-dynamic-with-proxy/cn.toml.base

@@ -47,3 +47,10 @@ type = "distributed-tae"
 port = %d
 unix-socket = "/tmp/mysql%d.sock"
 proxy-enabled = true
+
+[cn.frontend.iceberg]
+enable = true
+enable-write = true
+enable-delete = true
+enable-dml = true
+enable-maintenance = true

etc/launch-minio-local/.gitignore

@@ -3,3 +3,6 @@ minio-data/
 
 # MatrixOne data directory (if created in this directory)
 mo-data/
+
+# Generated local Iceberg profile environments
+*.generated.env