From 9aaad115d78491a20a4247122496203ba09de22b Mon Sep 17 00:00:00 2001 From: Vaibhav Joshi Date: Mon, 29 Jun 2026 20:26:13 +0530 Subject: [PATCH 1/2] Cloud Storage Support(#3079) - Implemented Pluggable Cloud Storage with S3 support out of the box --- docker/README.md | 8 + docker/cloud-storage/.gitignore | 5 + docker/cloud-storage/README.md | 612 ++++++++++++++++++ docker/cloud-storage/docker-compose.yml | 237 +++++++ .../entrypoints/pd-entrypoint.sh | 56 ++ .../entrypoints/store-entrypoint.sh | 93 +++ docker/cloud-storage/images/pd.Dockerfile | 29 + docker/cloud-storage/images/store.Dockerfile | 31 + .../scripts/test-graph-queries-and-sst.sh | 378 +++++++++++ docker/docker-compose.dev.yml | 2 +- .../pluggable-cloud-storage-architecture.md | 574 ++++++++++++++++ hugegraph-store/hg-store-cloud-s3/pom.xml | 82 +++ .../cloud/s3/S3CloudStorageProvider.java | 250 +++++++ ...hugegraph.store.cloud.CloudStorageProvider | 2 + .../store/cloud/CloudStorageConfig.java | 102 +++ .../store/cloud/CloudStorageProvider.java | 107 +++ .../cloud/CloudStorageProviderFactory.java | 178 +++++ .../src/assembly/static/conf/application.yml | 31 + hugegraph-store/hg-store-node/pom.xml | 4 + .../hugegraph/store/node/AppConfig.java | 96 +++ .../node/cloud/CloudStorageEventListener.java | 417 ++++++++++++ .../src/main/resources/application.yml | 14 + .../cloud/CloudStorageEventListenerTest.java | 392 +++++++++++ .../rocksdb/access/RocksDBFactory.java | 158 ++++- .../rocksdb/access/RocksDBSession.java | 6 + .../rocksdb/access/SessionOperatorImpl.java | 9 +- hugegraph-store/pom.xml | 15 +- install-dist/release-docs/LICENSE | 45 ++ install-dist/release-docs/NOTICE | 32 + .../LICENSE-reactive-streams-1.0.4.txt | 17 + .../scripts/dependency/known-dependencies.txt | 45 ++ .../regenerate_known_dependencies.sh | 0 pom.xml | 7 + 33 files changed, 4030 insertions(+), 4 deletions(-) create mode 100644 docker/cloud-storage/.gitignore create mode 100644 docker/cloud-storage/README.md create mode 100644 docker/cloud-storage/docker-compose.yml create mode 100755 docker/cloud-storage/entrypoints/pd-entrypoint.sh create mode 100755 docker/cloud-storage/entrypoints/store-entrypoint.sh create mode 100644 docker/cloud-storage/images/pd.Dockerfile create mode 100644 docker/cloud-storage/images/store.Dockerfile create mode 100755 docker/cloud-storage/scripts/test-graph-queries-and-sst.sh create mode 100644 hugegraph-store/docs/pluggable-cloud-storage-architecture.md create mode 100644 hugegraph-store/hg-store-cloud-s3/pom.xml create mode 100644 hugegraph-store/hg-store-cloud-s3/src/main/java/org/apache/hugegraph/store/cloud/s3/S3CloudStorageProvider.java create mode 100644 hugegraph-store/hg-store-cloud-s3/src/main/resources/META-INF/services/org.apache.hugegraph.store.cloud.CloudStorageProvider create mode 100644 hugegraph-store/hg-store-common/src/main/java/org/apache/hugegraph/store/cloud/CloudStorageConfig.java create mode 100644 hugegraph-store/hg-store-common/src/main/java/org/apache/hugegraph/store/cloud/CloudStorageProvider.java create mode 100644 hugegraph-store/hg-store-common/src/main/java/org/apache/hugegraph/store/cloud/CloudStorageProviderFactory.java create mode 100644 hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/cloud/CloudStorageEventListener.java create mode 100644 hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/cloud/CloudStorageEventListenerTest.java create mode 100644 install-dist/release-docs/licenses/LICENSE-reactive-streams-1.0.4.txt mode change 100644 => 100755 install-dist/scripts/dependency/regenerate_known_dependencies.sh diff --git a/docker/README.md b/docker/README.md index 9bc21b1ba7..0e9c8bb700 100644 --- a/docker/README.md +++ b/docker/README.md @@ -48,6 +48,14 @@ cd docker docker compose -f docker-compose.dev.yml up -d ``` +To run Store with the JDK 17 image profile (useful for ARM64 JNI-heavy workloads): + +```bash +cd docker +HG_STORE_DOCKERFILE=hugegraph-store/Dockerfile-jdk17 \ + docker compose -f docker-compose.dev.yml up -d --build +``` + - Images: built from source via `build: context: ..` with Dockerfiles - No `pull_policy` — builds locally, doesn't pull - Entrypoint scripts are baked into the built image (no volume mounts) diff --git a/docker/cloud-storage/.gitignore b/docker/cloud-storage/.gitignore new file mode 100644 index 0000000000..56a3ee1594 --- /dev/null +++ b/docker/cloud-storage/.gitignore @@ -0,0 +1,5 @@ +.artifacts/ +logs/ +data/ +.generated/ + diff --git a/docker/cloud-storage/README.md b/docker/cloud-storage/README.md new file mode 100644 index 0000000000..8e90dda0b3 --- /dev/null +++ b/docker/cloud-storage/README.md @@ -0,0 +1,612 @@ +# Cloud Storage (MinIO + HStore) Local Stack + +This stack runs: + +- MinIO (S3-compatible storage) +- 1 PD node +- 3 Store nodes +- `hg-store-cloud-s3` plugin loaded from local build artifacts + +The goal is to validate that RocksDB SST file events are mirrored to S3. + +## Prerequisites + +- Docker + Docker Compose +- Java 11+ +- Maven 3.5+ +- `curl` and `jq` (optional, for manual API testing) + +From repo root, build all modules: + +```bash +mvn clean package -DskipTests +``` + +This builds the necessary artifacts for the Docker images. + +## Quick Start + +Detect repo root first, then run the script: + +```bash +export REPO_ROOT="$(git rev-parse --show-toplevel)" +test -d "${REPO_ROOT}/docker/cloud-storage" + +cd "${REPO_ROOT}/docker/cloud-storage" +chmod +x ./scripts/*.sh ./entrypoints/*.sh +./scripts/test-graph-queries-and-sst.sh + +# For manual testing steps, keep the stack running after test +./scripts/test-graph-queries-and-sst.sh --keep-stack + +# For manual graph creation and testing (infrastructure only, no auto-load) +./scripts/test-graph-queries-and-sst.sh --no-load +``` + +## Test Output Files + +After running `make test` or `./scripts/test-graph-queries-and-sst.sh`, check `.generated/` for: + +- **`FULL-TEST-REPORT.txt`** — Complete test summary +- **`test-report.txt`** — Graph query test results +- **`minio-verification.txt`** — MinIO S3 object listing and SST count +- **`store-cluster.log`** — All store node logs with RocksDB/S3 events +- **`cli-load.log`** — Data load job output from hg-store-cli +- **`load-data.tsv`** — Generated test data file (200k entries) + +## Manual Verification Steps + +**Prerequisites:** Complete Step 1 first and wait for the infrastructure ready message. + +For hands-on validation with manual graph creation and queries, start the cluster using the automated script with `--keep-stack`, then follow interactive steps. **All commands use `$REPO_ROOT` to reference paths relative to the repository root.** + +**Note:** These steps verify end-to-end SST upload by: +1. Creating a graph schema +2. Adding test vertices and edges +3. Verifying data distribution across nodes +4. Restarting store nodes to flush SST files to RocksDB +5. Confirming SST files are uploaded to MinIO buckets + +### Step 0: Set Repository Root + +Before starting, set the `$REPO_ROOT` variable to point to the repository root. You can run this from anywhere: + +```bash +export REPO_ROOT=$(git rev-parse --show-toplevel) +``` + +Verify the variable is set correctly: + +```bash +echo $REPO_ROOT +``` + +### Step 1: Start Cluster with Infrastructure Only (No Auto-Load) + +Use the automated test script to build images, start the cluster, and **skip the automatic data load** so you can create your own graph structure: + +```bash +$REPO_ROOT/docker/cloud-storage/scripts/test-graph-queries-and-sst.sh --no-load +``` + +After Step 1 starts the stack, resolve the Docker network name for `docker run --network` commands: + +```bash +export HG_NET="${HG_NET:-cloud-storage-net}" +if docker network inspect "$HG_NET" >/dev/null 2>&1; then + echo "Using Docker network: $HG_NET" +elif docker network inspect cloud-storage-test_hg-net >/dev/null 2>&1; then + export HG_NET="cloud-storage-test_hg-net" + echo "Using Docker network: $HG_NET" +else + echo "No cloud-storage Docker network found. Ensure Step 1 completed successfully." +fi +``` + +This will: +- Build local artifacts (PD, Store, Store CLI, S3 plugin) +- Build Docker images +- Start MinIO + PD + 3 Store nodes +- Wait for all services to be healthy +- **Skip** initial data load phase +- **Keep the stack running** for manual steps + +The output will indicate when infrastructure is ready. + +### Step 2: Verify Cluster Health + +Verify all services are accessible. This may take a few minutes as the HugeGraph server needs time to initialize: + +```bash +# Wait for services to be healthy (may take 2-3 minutes on first run) +echo "Waiting for services to be healthy..." +for i in {1..60}; do + pd_ok=$(curl -fsS http://127.0.0.1:8620/v1/health >/dev/null 2>&1 && echo "yes" || echo "no") + store0_ok=$(curl -fsS http://127.0.0.1:8520/v1/health >/dev/null 2>&1 && echo "yes" || echo "no") + store1_ok=$(curl -fsS http://127.0.0.1:8521/v1/health >/dev/null 2>&1 && echo "yes" || echo "no") + store2_ok=$(curl -fsS http://127.0.0.1:8522/v1/health >/dev/null 2>&1 && echo "yes" || echo "no") + server_ok=$(curl -fsS http://127.0.0.1:8080/graphs >/dev/null 2>&1 && echo "yes" || echo "no") + + if [[ "$pd_ok" == "yes" && "$store0_ok" == "yes" && "$store1_ok" == "yes" && "$store2_ok" == "yes" && "$server_ok" == "yes" ]]; then + break + fi + + echo " Attempt $i/60: PD=$pd_ok Store0=$store0_ok Store1=$store1_ok Store2=$store2_ok Server=$server_ok" + sleep 2 +done + +# Final verification +curl -fsS http://127.0.0.1:8620/v1/health >/dev/null 2>&1 && echo "✓ PD OK" || echo "✗ PD FAILED" +curl -fsS http://127.0.0.1:8520/v1/health >/dev/null 2>&1 && echo "✓ Store0 OK" || echo "✗ Store0 FAILED" +curl -fsS http://127.0.0.1:8521/v1/health >/dev/null 2>&1 && echo "✓ Store1 OK" || echo "✗ Store1 FAILED" +curl -fsS http://127.0.0.1:8522/v1/health >/dev/null 2>&1 && echo "✓ Store2 OK" || echo "✗ Store2 FAILED" +curl -fsS http://127.0.0.1:8080/graphs >/dev/null 2>&1 && echo "✓ Server OK" || echo "✗ Server FAILED" +``` + +If all show "✓ OK", proceed to Step 3. If any show "✗ FAILED", check logs: +```bash +docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml logs --tail=50 server +``` + +**Note:** The `/graphs` endpoint returns a clean 200 OK response, making it the most reliable health check for the HugeGraph server. The `/gremlin` endpoint requires query parameters. + +### Step 3: Create Graph Schema + +Schema persists across restarts via rocksdb-cloud. If you see `ExistedException`, schema already exists. + +```bash +create_pk() { + local name="$1" dtype="$2" + local found=$(curl -s --compressed "http://localhost:8080/graphs/hugegraph/schema/propertykeys/$name" \ + | python3 -c "import sys,json; print(json.load(sys.stdin).get('name',''))" 2>/dev/null) + [[ "$found" == "$name" ]] && { echo " ✓ property key '$name' exists"; return; } + curl -s -X POST http://localhost:8080/graphs/hugegraph/schema/propertykeys \ + -H 'Content-Type: application/json' \ + -d "{\"name\":\"$name\",\"data_type\":\"$dtype\",\"cardinality\":\"SINGLE\"}" \ + | python3 -c "import sys,json; d=json.load(sys.stdin); print(' ✓ created property key:', d.get('property_key',{}).get('name','?'))" +} + +create_vl() { + local name="$1" props="$2" + local found=$(curl -s --compressed "http://localhost:8080/graphs/hugegraph/schema/vertexlabels/$name" \ + | python3 -c "import sys,json; print(json.load(sys.stdin).get('name',''))" 2>/dev/null) + [[ "$found" == "$name" ]] && { echo " ✓ vertex label '$name' exists"; return; } + curl -s -X POST http://localhost:8080/graphs/hugegraph/schema/vertexlabels \ + -H 'Content-Type: application/json' \ + -d "{\"name\":\"$name\",\"id_strategy\":\"AUTOMATIC\",\"properties\":$props}" \ + | python3 -c "import sys,json; d=json.load(sys.stdin); print(' ✓ created vertex label:', d.get('name','?'))" +} + +create_pk "name" "TEXT" +create_pk "age" "INT" +create_pk "city" "TEXT" + +create_vl "person" '["name","age","city"]' +create_vl "location" '["name"]' + +FOUND_EL=$(curl -s --compressed http://localhost:8080/graphs/hugegraph/schema/edgelabels/lives_in \ + | python3 -c "import sys,json; print(json.load(sys.stdin).get('name',''))" 2>/dev/null) +if [[ "$FOUND_EL" == "lives_in" ]]; then + echo " ✓ edge label 'lives_in' exists" +else + curl -s -X POST http://localhost:8080/graphs/hugegraph/schema/edgelabels \ + -H 'Content-Type: application/json' \ + -d '{"name":"lives_in","source_label":"person","target_label":"location"}' \ + | python3 -c "import sys,json; d=json.load(sys.stdin); print(' ✓ created edge label:', d.get('name','?'))" +fi + +echo "✓ Schema ready" +``` + +### Step 4: Add Vertices and Edges + +Add sufficient data to trigger RocksDB compaction (minimum 150+ vertices): + +```bash +insert_vertex() { + local label="$1" props="$2" + local tmpfile=$(mktemp) + local code=$(curl -s -o "$tmpfile" -w "%{http_code}" -X POST \ + http://localhost:8080/graphs/hugegraph/graph/vertices \ + -H 'Content-Type: application/json' \ + -d "{\"label\":\"$label\",\"properties\":$props}") + local body=$(cat "$tmpfile") + rm -f "$tmpfile" + if [[ "$code" == "201" || "$code" == "200" ]]; then + echo "$body" | python3 -c "import sys,json; print(json.load(sys.stdin)['id'])" + else + echo "ERROR: HTTP $code - $body" >&2 + echo "" + fi +} + +echo "Inserting test vertices (150+)..." +for i in {1..150}; do + age=$((20 + i % 50)) + city=$((i % 5)) + vid=$(insert_vertex "person" "{\"name\":\"person_$i\",\"age\":$age,\"city\":\"city_$city\"}") + if (( i % 50 == 0 )); then + echo " ✓ Inserted $i vertices" + fi +done + +echo "Adding location vertices..." +for i in {0..4}; do + insert_vertex "location" "{\"name\":\"city_$i\"}" >/dev/null +done + +echo "✓ Inserted 150+ test vertices (sufficient for RocksDB compaction)" +``` + +**Why 150+ vertices?** Smaller datasets may not trigger RocksDB compaction. 150+ vertices across 3 store nodes ensures enough write activity to generate SST files. + +### Step 5: Execute Graph Queries (Optional) + +Verify the data was stored: + +```bash +echo "=== Vertex count ===" +curl -s --compressed http://localhost:8080/graphs/hugegraph/graph/vertices | python3 -c "import sys,json; data=json.load(sys.stdin); print('Total vertices:', len(data.get('vertices',[])))" + +echo "=== Sample vertex ===" +curl -s --compressed http://localhost:8080/graphs/hugegraph/graph/vertices | python3 -c "import sys,json; data=json.load(sys.stdin); vertices=data.get('vertices', []); print(json.dumps(vertices[0], indent=2) if vertices else 'No vertices')" | head -10 +``` + +**Note:** Querying 150+ vertices can return large result sets. To keep this step fast, the above commands just check the count and show a sample. + +### Step 6: Verify Data Distribution + +Verify data has been distributed across store nodes: + +```bash +echo "=== Partition Info (before flush) ===" +for i in 0 1 2; do + port=$((8520 + i)) + echo "" + echo "Store$i (port $port):" + curl -s http://127.0.0.1:$port/v1/partitions | python3 -c "import sys,json; data=json.load(sys.stdin); print(f' Partitions: {len(data.get(\"partitions\",[])) if isinstance(data.get(\"partitions\"), list) else \"N/A\"}')" 2>/dev/null || echo " (unable to retrieve)" +done +``` + +**Expected:** Each store should show partition information, indicating data distribution across the cluster. + +### Step 7: Flush Data to S3 (Restart Store Nodes) + +To trigger SST file uploads to S3, restart the Store nodes. This forces RocksDB to: +1. Flush in-memory data to disk as SST files +2. Trigger the cloud storage event listener +3. Upload SST files to MinIO buckets + +```bash +echo "Restarting store nodes to flush data to S3..." +# Use container names so this works for both static compose and generated test compose +docker restart cloud-storage-store0 cloud-storage-store1 cloud-storage-store2 + +echo "Waiting for stores to restart and flush..." +sleep 20 + +echo "Verifying store health after restart..." +for i in 0 1 2; do + port=$((8520 + i)) + curl -fsS http://127.0.0.1:$port/v1/health >/dev/null 2>&1 && echo "✓ Store$i OK" || echo "✗ Store$i FAILED" +done +``` + +**What happens behind the scenes:** +- Docker restarts the store containers +- RocksDB initializes and detects in-memory data +- RocksDB writes all data as SST files to disk +- Cloud storage listener detects flush events +- SST files are uploaded to MinIO in parallel +- Stores become healthy and rejoin cluster + +### Step 8: Final S3 Verification (After Flush) + +Verify that SST files have been successfully uploaded to MinIO buckets: + +```bash +echo "=== Checking MinIO buckets after flush ===" +docker run --rm --entrypoint /bin/sh --network "$HG_NET" minio/mc:RELEASE.2025-08-13T08-35-41Z \ + -c 'mc alias set local http://minio:9000 minioadmin minioadmin >/dev/null && \ + for b in hugegraph-store0 hugegraph-store1 hugegraph-store2; do \ + echo ""; \ + echo "### Bucket: $b"; \ + total_count=$(mc ls --recursive local/$b | wc -l); \ + sst_count=$(mc find local/$b --name "*.sst" | wc -l); \ + printf " Total objects: %d\n" "$total_count"; \ + printf " SST files: %d\n" "$sst_count"; \ + if (( sst_count > 0 )); then \ + echo " Sample SST files (first 3):"; \ + mc find local/$b --name "*.sst" | head -3; \ + fi; \ + done' +``` + +**What successful upload looks like:** +``` +### Bucket: hugegraph-store0 + Total objects: 8 + SST files: 6 + Sample SST files (first 3): + local/hugegraph-store0/partition-1/0000000001.sst + local/hugegraph-store0/partition-1/0000000002.sst + local/hugegraph-store0/partition-1/manifest + +### Bucket: hugegraph-store1 + Total objects: 9 + SST files: 7 + ... + +### Bucket: hugegraph-store2 + Total objects: 8 + SST files: 6 + ... +``` + +**Success criteria:** +- ✅ All three buckets show `Total objects: > 0` +- ✅ All three buckets show `SST files: > 0` +- ✅ SST counts are roughly balanced across buckets +- ✅ No errors from mc command + +**If buckets are still empty after flush:** +1. Check store logs for upload errors: `docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml logs store0 | grep -i "s3\|cloud\|error"` +2. Verify cloud storage was initialized: `docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml logs store0 | grep -i "Cloud storage provider.*initialized\|S3CloudStorageProvider initialized"` +3. Check that data was written: `docker exec cloud-storage-store0 sh -lc 'find /hugegraph-store/storage -name "*.sst" | wc -l'` + +### Step 9 (Optional): Destroy the Cluster + +When done with manual verification, stop and clean up: + +```bash +docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml down +``` + +To also remove data and logs directories: + +```bash +docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml down +rm -rf $REPO_ROOT/docker/cloud-storage/data/ $REPO_ROOT/docker/cloud-storage/logs/ +``` + +To reset everything including built Docker images (for a fresh start): + +```bash +docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml down +rm -rf $REPO_ROOT/docker/cloud-storage/data/ $REPO_ROOT/docker/cloud-storage/logs/ $REPO_ROOT/docker/cloud-storage/.artifacts/ +docker rmi \ + hugegraph-pd-cloud-storage:local \ + hugegraph-store-cloud-storage:local \ + 2>/dev/null || true +``` + + +## Architecture Notes + +- **Entrypoints**: Store containers inject `cloud.storage.*` config via `SPRING_APPLICATION_JSON` in `entrypoints/store-entrypoint.sh`. +- **Config precedence**: values injected by `SPRING_APPLICATION_JSON` override defaults in `/hugegraph-store/conf/application.yml`. +- **Plugin Loading**: Store startup uses `PropertiesLauncher` with `-Dloader.path=/hugegraph-store/plugins` to discover S3 provider JAR via `ServiceLoader` +- **Logging**: Store containers run with `DEBUG` level for: + - `org.apache.hugegraph.store.node.cloud` + - `org.apache.hugegraph.rocksdb.access` +- **Networking**: Containers run on `cloud-storage-net` (static compose) or `cloud-storage-test_hg-net` (default generated test stack) +- **Storage**: Local bind mounts under `data/` and `logs/` for inspection +- **Bucket layout**: each Store node writes SSTs to a dedicated bucket (`store0 -> hugegraph-store0`, `store1 -> hugegraph-store1`, `store2 -> hugegraph-store2`) + +### Troubleshooting Manual Verification Steps + +**Step 2: Server not becoming healthy** +```bash +# Check if server container is running +docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml ps server + +# Check server logs +docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml logs --tail=100 server + +# Verify port 8080 is accessible and /graphs endpoint works +curl -v http://127.0.0.1:8080/graphs +``` + +**Note:** The server health check uses `/graphs` endpoint (HTTP 200 OK), not `/gremlin` (requires query parameter). + +**Step 3-4: Graph API errors (HTTP errors)** +```bash +# Verify server is healthy +curl -fsS http://127.0.0.1:8080/graphs && echo "Server OK" || echo "Server not ready" + +# Check if graph exists +curl -s http://localhost:8080/graphs/hugegraph/graph/vertices | python3 -c "import sys,json; print(json.load(sys.stdin).keys())" + +# Check server logs for errors +docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml logs server | grep -i "error\|exception" | tail -20 +``` + +**Step 6: Partition info showing empty** +```bash +# Data may still be in RocksDB memory, not yet in partitions +# This is normal - proceed to Step 7 to flush + +# Or check if data was actually written +docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml logs store0 | grep -i "vertex\|edge\|insert" | tail -10 +``` + +**Step 7: Stores not coming back healthy after restart** +```bash +# Check individual store health +for i in 0 1 2; do + port=$((8520 + i)) + echo "Store$i:" + curl -v http://127.0.0.1:$port/v1/health 2>&1 | grep -E "HTTP/|Connection refused" +done + +# Check store logs for startup errors +docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml logs store0 | tail -50 | grep -i "error\|exception" +``` + +**Step 8: Buckets showing 0 files after flush** + +This is the most common issue. Debug step-by-step: + +1. **Verify stores have data on disk:** + ```bash + docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml exec store0 \ + find /hugegraph-store/storage -name "*.sst" | wc -l + ``` + If output is 0, no SST files were created. Go back to Step 4 and ensure data was inserted. + +2. **Verify cloud storage was initialized:** + ```bash + docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml logs store0 | \ + grep -i "S3CloudStorageProvider\|cloud storage" | head -5 + ``` + If no output, plugin didn't load. Check `/hugegraph-store/plugins/` for `hg-store-cloud-s3.jar`. + +3. **Check for S3 upload errors:** + ```bash + docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml logs store0 | \ + grep -i "upload\|s3\|error" | tail -20 + ``` + +4. **Verify MinIO connectivity:** + ```bash + docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml exec store0 \ + curl -v http://minio:9000/minio/health/live + ``` + +5. **Manually check MinIO buckets:** + ```bash + export HG_NET="cloud-storage-net" + docker run --rm --entrypoint /bin/sh --network "$HG_NET" minio/mc:RELEASE.2025-08-13T08-35-41Z \ + -c 'mc alias set local http://minio:9000 minioadmin minioadmin && mc ls local/' + ``` + +### Troubleshooting: General Infrastructure Issues + +### Store crashes with SIGSEGV on ARM64 (libjvm.so) + +If store logs show a fatal JVM crash like: + +```text +SIGSEGV ... libjvm.so ... linux-aarch64 +``` + +this is typically seen on ARM64 hosts with JVM + RocksDB startup. + +**On ARM64 hosts, the script automatically applies safe JVM defaults:** +- Uses Java 17 runtime image (not Java 11) +- Applies conservative JVM flags: `-XX:+UseSerialGC -XX:-UseCompressedOops -XX:-UseCompressedClassPointers` + +Simply run without special flags: + +```bash +$REPO_ROOT/docker/cloud-storage/scripts/test-graph-queries-and-sst.sh --no-load +``` + +The script detects your host architecture and applies appropriate defaults automatically. + +### Custom overrides on ARM64 + +If you need different JVM flags or runtime images: + +```bash +HG_DOCKER_DEFAULT_PLATFORM=linux/amd64 \ +HG_PD_JAVA_RUNTIME_IMAGE=eclipse-temurin:17-jre \ +HG_STORE_JAVA_RUNTIME_IMAGE=eclipse-temurin:17-jre \ +HG_STORE_JAVA_OPTS="-XX:+UseSerialGC -XX:-UseCompressedOops -XX:-UseCompressedClassPointers" \ +$REPO_ROOT/docker/cloud-storage/scripts/test-graph-queries-and-sst.sh --no-load +``` + +But in most cases, running without overrides should work: + +```bash +$REPO_ROOT/docker/cloud-storage/scripts/test-graph-queries-and-sst.sh --no-load +``` + +If old containers are still running, force recreate with fresh env: + +```bash +docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml down +$REPO_ROOT/docker/cloud-storage/scripts/test-graph-queries-and-sst.sh --no-load + +# Verify container runtime JDK actually changed +docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml run --rm --entrypoint java store0 -version +``` + +If the crash persists, inspect the generated JVM error file from store logs/data path: + +```bash +docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml logs store0 | tail -200 +docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml exec store0 ls -lah /hugegraph-store/hs_err_pid*.log +``` + +### No SST files appear in MinIO + +- Short test runs may not trigger compaction; SST deletion especially requires heavier/longer load +- Check store logs for upload errors: + ```bash + docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml logs store0 | grep -i "cloud\|s3" + ``` +- Verify MinIO is accessible: + ```bash + docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml exec minio mc ls local/ 2>&1 | head -20 + ``` + +### Query tests show FAILED + +- Ensure all store nodes are healthy: check `/v1/health` endpoints +- Manually test: `curl -fsS http://127.0.0.1:8520/v1/partitions | jq` +- Check store logs with: + ```bash + docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml logs store0 store1 store2 + ``` + +### Keep stack running for debugging + +```bash +# Run test but keep stack up +KEEP_STACK=1 $REPO_ROOT/docker/cloud-storage/scripts/test-graph-queries-and-sst.sh +``` + +Then inspect manually: +```bash +docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml logs store0 store1 store2 +docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml exec store0 ls -la /hugegraph-store/storage +``` + +Stop manually when done: +```bash +docker compose -f $REPO_ROOT/docker/cloud-storage/docker-compose.yml down +``` + +## Summary of Manual Verification Workflow + +This manual verification process validates the complete SST upload pipeline: + +1. **Step 1:** Start the infrastructure (MinIO, PD, 3 Store nodes, HugeGraph Server) +2. **Step 2:** Wait for all services to become healthy +3. **Step 3:** Create graph schema (property keys, vertex labels, edge labels) +4. **Step 4:** Load 150+ test vertices to trigger compaction +5. **Step 5:** (Optional) Verify data was stored +6. **Step 6:** Verify data distribution across store nodes +7. **Step 7:** Restart store nodes to flush SST files to disk and upload to MinIO +8. **Step 8:** Verify SST files are present in MinIO buckets +9. **Step 9:** Cleanup when done + +**Success = Non-zero SST file counts in all three buckets after Step 8** + +## What Gets Verified + +✅ Graph schema creation works +✅ Vertex/edge insertion works +✅ Data distribution across 3 store nodes +✅ RocksDB SST file generation on restart +✅ Cloud storage plugin uploads SST files to MinIO +✅ Multiple buckets receive files consistently + +## Notes + +- S3 provider JAR must be in `/hugegraph-store/plugins/` for `ServiceLoader` discovery +- Plugin dependency staging filters extra SLF4J binding jars to avoid duplicate binding warnings at runtime +- Cloud settings are read from environment variables at store startup time +- **Minimum data size:** 150+ vertices is recommended to trigger RocksDB SST file generation. Very small datasets may not create any SST files. +- MinIO buckets are pre-created by `minio-init` service: `hugegraph-store0`, `hugegraph-store1`, `hugegraph-store2` diff --git a/docker/cloud-storage/docker-compose.yml b/docker/cloud-storage/docker-compose.yml new file mode 100644 index 0000000000..269fc2c3c7 --- /dev/null +++ b/docker/cloud-storage/docker-compose.yml @@ -0,0 +1,237 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: cloud-storage + +networks: + hg-net: + name: cloud-storage-net + driver: bridge + +volumes: + minio-data: + +services: + minio: + image: minio/minio:latest + container_name: cloud-storage-minio + command: server /data --console-address ':9001' + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + ports: + - "9000:9000" + - "9001:9001" + volumes: + - minio-data:/data + networks: [hg-net] + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 10s + timeout: 5s + retries: 20 + + minio-init: + image: minio/mc:RELEASE.2025-08-13T08-35-41Z + container_name: cloud-storage-minio-init + depends_on: + minio: + condition: service_healthy + entrypoint: > + /bin/sh -c ' + mc alias set local http://minio:9000 minioadmin minioadmin && + mc mb -p local/hugegraph-store0 || true && + mc mb -p local/hugegraph-store1 || true && + mc mb -p local/hugegraph-store2 || true && + mc anonymous set none local/hugegraph-store0 || true && + mc anonymous set none local/hugegraph-store1 || true && + mc anonymous set none local/hugegraph-store2 || true + ' + networks: [hg-net] + + pd: + build: + context: ../.. + dockerfile: docker/cloud-storage/images/pd.Dockerfile + args: + JAVA_RUNTIME_IMAGE: ${HG_PD_JAVA_RUNTIME_IMAGE:-eclipse-temurin:11-jre} + image: hugegraph/pd:cloud-storage-local + container_name: cloud-storage-pd + depends_on: + minio-init: + condition: service_completed_successfully + environment: + HG_PD_GRPC_HOST: pd + HG_PD_GRPC_PORT: "8686" + HG_PD_REST_PORT: "8620" + HG_PD_RAFT_ADDRESS: pd:8610 + HG_PD_RAFT_PEERS_LIST: pd:8610 + HG_PD_INITIAL_STORE_LIST: store0:8500,store1:8500,store2:8500 + HG_PD_DATA_PATH: /hugegraph-pd/pd_data + HG_PD_INITIAL_STORE_COUNT: "3" + ports: + - "8620:8620" + - "8686:8686" + volumes: + - ./data/pd:/hugegraph-pd/pd_data + - ./logs/pd:/hugegraph-pd/logs + networks: [hg-net] + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8620/v1/health >/dev/null || exit 1"] + interval: 15s + timeout: 10s + retries: 30 + start_period: 60s + + store0: + build: + context: ../.. + dockerfile: docker/cloud-storage/images/store.Dockerfile + args: + JAVA_RUNTIME_IMAGE: ${HG_STORE_JAVA_RUNTIME_IMAGE:-eclipse-temurin:11-jre} + image: hugegraph/store:cloud-storage-local + container_name: cloud-storage-store0 + depends_on: + pd: + condition: service_healthy + environment: + HG_STORE_PD_ADDRESS: pd:8686 + HG_STORE_GRPC_HOST: store0 + HG_STORE_GRPC_PORT: "8500" + HG_STORE_REST_PORT: "8520" + HG_STORE_RAFT_ADDRESS: store0:8510 + HG_CLOUD_STORAGE_ENDPOINT: http://minio:9000 + HG_CLOUD_STORAGE_BUCKET: hugegraph-store0 + JAVA_OPTS: ${HG_STORE_JAVA_OPTS:--Xms4g -Xmx4g -XX:+UseG1GC} + ports: + - "8500:8500" + - "8510:8510" + - "8520:8520" + volumes: + - ./data/store0:/hugegraph-store/storage + - ./logs/store0:/hugegraph-store/logs + networks: [hg-net] + deploy: + resources: + limits: + memory: 8G + reservations: + memory: 6G + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8520/v1/health >/dev/null || exit 1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 120s + + store1: + image: hugegraph/store:cloud-storage-local + container_name: cloud-storage-store1 + depends_on: + pd: + condition: service_healthy + environment: + HG_STORE_PD_ADDRESS: pd:8686 + HG_STORE_GRPC_HOST: store1 + HG_STORE_GRPC_PORT: "8500" + HG_STORE_REST_PORT: "8520" + HG_STORE_RAFT_ADDRESS: store1:8510 + HG_CLOUD_STORAGE_ENDPOINT: http://minio:9000 + HG_CLOUD_STORAGE_BUCKET: hugegraph-store1 + JAVA_OPTS: ${HG_STORE_JAVA_OPTS:--Xms4g -Xmx4g -XX:+UseG1GC} + ports: + - "8501:8500" + - "8511:8510" + - "8521:8520" + volumes: + - ./data/store1:/hugegraph-store/storage + - ./logs/store1:/hugegraph-store/logs + networks: [hg-net] + deploy: + resources: + limits: + memory: 8G + reservations: + memory: 6G + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8520/v1/health >/dev/null || exit 1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 120s + + store2: + image: hugegraph/store:cloud-storage-local + container_name: cloud-storage-store2 + depends_on: + pd: + condition: service_healthy + environment: + HG_STORE_PD_ADDRESS: pd:8686 + HG_STORE_GRPC_HOST: store2 + HG_STORE_GRPC_PORT: "8500" + HG_STORE_REST_PORT: "8520" + HG_STORE_RAFT_ADDRESS: store2:8510 + HG_CLOUD_STORAGE_ENDPOINT: http://minio:9000 + HG_CLOUD_STORAGE_BUCKET: hugegraph-store2 + JAVA_OPTS: ${HG_STORE_JAVA_OPTS:--Xms4g -Xmx4g -XX:+UseG1GC} + ports: + - "8502:8500" + - "8512:8510" + - "8522:8520" + volumes: + - ./data/store2:/hugegraph-store/storage + - ./logs/store2:/hugegraph-store/logs + networks: [hg-net] + deploy: + resources: + limits: + memory: 8G + reservations: + memory: 6G + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8520/v1/health >/dev/null || exit 1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 120s + + server: + image: hugegraph/server:1.7.0 + container_name: cloud-storage-server + depends_on: + store0: + condition: service_healthy + store1: + condition: service_healthy + store2: + condition: service_healthy + environment: + GREMLIN_SERVER_HOST: 0.0.0.0 + GREMLIN_SERVER_PORT: "8182" + ports: + - "8080:8080" + - "8182:8182" + volumes: + - ./logs/server:/hugegraph-server/logs + networks: [hg-net] + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8080/gremlin >/dev/null || exit 1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 120s diff --git a/docker/cloud-storage/entrypoints/pd-entrypoint.sh b/docker/cloud-storage/entrypoints/pd-entrypoint.sh new file mode 100755 index 0000000000..8ad2159d5e --- /dev/null +++ b/docker/cloud-storage/entrypoints/pd-entrypoint.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail + +: "${HG_PD_GRPC_HOST:=pd}" +: "${HG_PD_GRPC_PORT:=8686}" +: "${HG_PD_REST_PORT:=8620}" +: "${HG_PD_RAFT_ADDRESS:=pd:8610}" +: "${HG_PD_RAFT_PEERS_LIST:=pd:8610}" +: "${HG_PD_DATA_PATH:=/hugegraph-pd/pd_data}" +: "${HG_PD_INITIAL_STORE_LIST:=store0:8500,store1:8500,store2:8500}" +: "${HG_PD_INITIAL_STORE_COUNT:=3}" + +json_escape() { + local s="$1" + s=${s//\\/\\\\} + s=${s//\"/\\\"} + s=${s//$'\n'/} + printf "%s" "$s" +} + +export SPRING_APPLICATION_JSON="$(cat <&2 + exit 2 + fi +} + +json_escape() { + local s="$1" + s=${s//\\/\\\\} + s=${s//\"/\\\"} + s=${s//$'\n'/} + printf "%s" "$s" +} + +require_env "HG_STORE_PD_ADDRESS" +require_env "HG_STORE_GRPC_HOST" +require_env "HG_STORE_RAFT_ADDRESS" + +: "${HG_STORE_GRPC_PORT:=8500}" +: "${HG_STORE_REST_PORT:=8520}" +: "${HG_STORE_DATA_PATH:=/hugegraph-store/storage}" +: "${HG_CLOUD_STORAGE_PROVIDER:=s3}" +: "${HG_CLOUD_STORAGE_ENABLED:=true}" +: "${HG_CLOUD_STORAGE_BUCKET:=hugegraph-store}" +: "${HG_CLOUD_STORAGE_REGION:=us-east-1}" +: "${HG_CLOUD_STORAGE_ENDPOINT:=http://minio:9000}" +: "${HG_CLOUD_STORAGE_ACCESS_KEY:=minioadmin}" +: "${HG_CLOUD_STORAGE_SECRET_KEY:=minioadmin}" +: "${HG_CLOUD_STORAGE_PATH_PREFIX:=hugegraph}" + +export SPRING_APPLICATION_JSON="$(cat </dev/null 2>&1 || { echo "ERROR: $1 not found" >&2; exit 2; }; } + +find_dist_dir() { + local glob="$1" + local d + for d in $glob; do + [[ -d "$d" ]] && { echo "$d"; return 0; } + done + return 1 +} + +find_plugin_jar() { + local f + for f in "${REPO_ROOT}"/hugegraph-store/hg-store-cloud-s3/target/hg-store-cloud-s3-*.jar; do + [[ -f "$f" ]] || continue + case "$f" in + *-sources.jar|*-javadoc.jar|*original*) continue ;; + esac + echo "$f" + return 0 + done + return 1 +} + +prepare_artifacts() { + local pd_src store_src plugin_jar plugin_dep_dir dep_base + + pd_src="$(find_dist_dir "${REPO_ROOT}/hugegraph-pd/apache-hugegraph-pd-*")" || { + echo "ERROR: PD dist not found under ${REPO_ROOT}/hugegraph-pd/apache-hugegraph-pd-*" >&2 + echo "Run: mvn clean package -DskipTests" >&2 + exit 2 + } + store_src="$(find_dist_dir "${REPO_ROOT}/hugegraph-store/apache-hugegraph-store-*")" || { + echo "ERROR: Store dist not found under ${REPO_ROOT}/hugegraph-store/apache-hugegraph-store-*" >&2 + echo "Run: mvn clean package -DskipTests" >&2 + exit 2 + } + plugin_jar="$(find_plugin_jar)" || { + echo "ERROR: S3 plugin jar not found under ${REPO_ROOT}/hugegraph-store/hg-store-cloud-s3/target/" >&2 + echo "Run: mvn clean package -DskipTests" >&2 + exit 2 + } + + log "preparing Docker artifacts in ${ARTIFACTS_DIR}" + rm -rf "${ARTIFACTS_DIR}" + mkdir -p "${ARTIFACTS_DIR}/pd-dist" "${ARTIFACTS_DIR}/store-dist" "${ARTIFACTS_DIR}/plugins" + + cp -R "${pd_src}/." "${ARTIFACTS_DIR}/pd-dist/" + cp -R "${store_src}/." "${ARTIFACTS_DIR}/store-dist/" + cp "${plugin_jar}" "${ARTIFACTS_DIR}/plugins/" + + plugin_dep_dir="${REPO_ROOT}/hugegraph-store/hg-store-cloud-s3/target/dependency" + if [[ -d "${plugin_dep_dir}" ]]; then + # Keep only external plugin deps; internal HugeGraph jars must come from /hugegraph-store/lib. + for dep in "${plugin_dep_dir}"/*.jar; do + [[ -f "${dep}" ]] || continue + dep_base="$(basename "${dep}")" + case "${dep_base}" in + hg-*.jar|hugegraph-*.jar) continue ;; + esac + cp "${dep}" "${ARTIFACTS_DIR}/plugins/" + done + else + log "warning: plugin dependency dir not found at ${plugin_dep_dir}; continuing with plugin jar only" + fi + + # Prevent duplicate SLF4J binding clashes from plugin dependency staging. + rm -f "${ARTIFACTS_DIR}"/plugins/log4j-slf4j-impl-*.jar "${ARTIFACTS_DIR}"/plugins/slf4j-log4j12-*.jar || true +} + +ensure_image() { + local img="$1" + docker image inspect "$img" >/dev/null 2>&1 && return 0 + # Check if it's a local build image (contains "cloud-storage-local") + if [[ "$img" == *"cloud-storage-local"* ]]; then + log "image $img is local-build only (will be built via docker compose build)" + return 0 + fi + log "pulling $img..." + docker pull "$img" >/dev/null || exit 3 +} +ensure_minio_buckets() { + for bucket in "$S3_BUCKET_STORE0" "$S3_BUCKET_STORE1" "$S3_BUCKET_STORE2"; do + docker run --rm --network "$1" --entrypoint /bin/sh "$MINIO_MC_IMAGE" -c "mc alias set local http://minio:9000 $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD >/dev/null && mc mb --ignore-existing local/$bucket >/dev/null" + done +} +wait_svc() { + local svc max=120 i=0 + svc="$1" + while [[ $i -lt $max ]]; do + local cid status + cid=$(docker compose -f "$COMPOSE_FILE" ps -q "$svc" 2>/dev/null || true) + [[ -z "$cid" ]] && { sleep 2; i=$((i+1)); continue; } + status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$cid" 2>/dev/null || true) + [[ "$status" == "healthy" || "$status" == "running" ]] && { log "✓ $svc"; return 0; } + sleep 2 + i=$((i+1)) + done + echo "ERROR: $svc timeout" >&2 + return 1 +} +wait_http() { + local url max=120 i=0 + url="$1" + while [[ $i -lt $max ]]; do + [[ $(curl -so /dev/null -w "%{http_code}" "$url" 2>/dev/null) == "200" ]] && { log "✓ $url ready"; return 0; } + sleep 2 + i=$((i+1)) + done + echo "ERROR: $url timeout" >&2 + return 1 +} +cleanup() { [[ "$KEEP_UP" == "true" ]] || (docker compose -f "$COMPOSE_FILE" down -v 2>/dev/null || true); } +trap cleanup EXIT +need_cmd docker curl python3 +log "pulling images..." +ensure_image "$MINIO_IMAGE" +ensure_image "$MINIO_MC_IMAGE" +ensure_image "$HG_PD_IMAGE" +ensure_image "$HG_STORE_IMAGE" +ensure_image "$HG_SERVER_IMAGE" +mkdir -p "$GENERATED_DIR" +cat > "$SERVER_GRAPH_CONF" << 'PROPS' +gremlin.graph=org.apache.hugegraph.HugeFactory +backend=hstore +serializer=binary +store=hugegraph +task.scheduler_type=local +pd.peers=pd:8686 +PROPS +mkdir -p "$(dirname "$COMPOSE_FILE")" +docker compose -f "$COMPOSE_FILE" down -v 2>/dev/null || true +log "generating docker-compose.yml..." +cat > "$COMPOSE_FILE" << 'YAML' +services: + minio: + image: minio/minio:latest + container_name: cloud-storage-minio + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + ports: ["9000:9000", "9001:9001"] + volumes: [hg-minio-data:/data] + networks: [hg-net] + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:9000/minio/health/live >/dev/null || exit 1"] + interval: 5s + timeout: 5s + retries: 40 + pd: + build: + context: ../../../ + dockerfile: docker/cloud-storage/images/pd.Dockerfile + image: hugegraph/pd:cloud-storage-local + container_name: cloud-storage-pd + hostname: pd + depends_on: + minio: + condition: service_healthy + environment: + HG_PD_GRPC_HOST: pd + HG_PD_GRPC_PORT: "8686" + HG_PD_REST_PORT: "8620" + HG_PD_RAFT_ADDRESS: pd:8610 + HG_PD_RAFT_PEERS_LIST: pd:8610 + HG_PD_INITIAL_STORE_LIST: store0:8500,store1:8500,store2:8500 + HG_PD_INITIAL_STORE_COUNT: "3" + HG_PD_DATA_PATH: /hugegraph-pd/pd_data + ports: ["8620:8620", "8686:8686"] + volumes: [hg-pd-data:/hugegraph-pd/pd_data] + networks: [hg-net] + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8620/v1/health >/dev/null || exit 1"] + interval: 10s + timeout: 5s + retries: 30 + store0: + build: + context: ../../../ + dockerfile: docker/cloud-storage/images/store.Dockerfile + image: hugegraph/store:cloud-storage-local + container_name: cloud-storage-store0 + hostname: store0 + depends_on: + pd: + condition: service_healthy + environment: + HG_STORE_PD_ADDRESS: pd:8686 + HG_STORE_GRPC_HOST: store0 + HG_STORE_GRPC_PORT: "8500" + HG_STORE_REST_PORT: "8520" + HG_STORE_RAFT_ADDRESS: store0:8510 + HG_STORE_DATA_PATH: /hugegraph-store/storage + HG_CLOUD_STORAGE_ENABLED: "true" + HG_CLOUD_STORAGE_BUCKET: "hugegraph-store0" + HG_CLOUD_STORAGE_ENDPOINT: "http://minio:9000" + HG_CLOUD_STORAGE_REGION: "us-east-1" + HG_CLOUD_STORAGE_ACCESS_KEY: "minioadmin" + HG_CLOUD_STORAGE_SECRET_KEY: "minioadmin" + HG_CLOUD_STORAGE_PATH_STYLE: "true" + HG_CLOUD_STORAGE_SYNC_INTERVAL_SECONDS: "30" + HG_CLOUD_STORAGE_SYNCHRONOUS_SST_UPLOAD_MODE: "true" + ports: ["8520:8520"] + volumes: [hg-store0-data:/hugegraph-store/storage] + networks: [hg-net] + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8520/v1/health >/dev/null || exit 1"] + interval: 10s + timeout: 10s + retries: 40 + start_period: 60s + store1: + build: + context: ../../../ + dockerfile: docker/cloud-storage/images/store.Dockerfile + image: hugegraph/store:cloud-storage-local + container_name: cloud-storage-store1 + hostname: store1 + depends_on: + pd: + condition: service_healthy + environment: + HG_STORE_PD_ADDRESS: pd:8686 + HG_STORE_GRPC_HOST: store1 + HG_STORE_GRPC_PORT: "8500" + HG_STORE_REST_PORT: "8520" + HG_STORE_RAFT_ADDRESS: store1:8510 + HG_STORE_DATA_PATH: /hugegraph-store/storage + HG_CLOUD_STORAGE_ENABLED: "true" + HG_CLOUD_STORAGE_BUCKET: "hugegraph-store1" + HG_CLOUD_STORAGE_ENDPOINT: "http://minio:9000" + HG_CLOUD_STORAGE_REGION: "us-east-1" + HG_CLOUD_STORAGE_ACCESS_KEY: "minioadmin" + HG_CLOUD_STORAGE_SECRET_KEY: "minioadmin" + HG_CLOUD_STORAGE_PATH_STYLE: "true" + HG_CLOUD_STORAGE_SYNC_INTERVAL_SECONDS: "30" + HG_CLOUD_STORAGE_SYNCHRONOUS_SST_UPLOAD_MODE: "true" + ports: ["8521:8520"] + volumes: [hg-store1-data:/hugegraph-store/storage] + networks: [hg-net] + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8520/v1/health >/dev/null || exit 1"] + interval: 10s + timeout: 10s + retries: 40 + start_period: 60s + store2: + build: + context: ../../../ + dockerfile: docker/cloud-storage/images/store.Dockerfile + image: hugegraph/store:cloud-storage-local + container_name: cloud-storage-store2 + hostname: store2 + depends_on: + pd: + condition: service_healthy + environment: + HG_STORE_PD_ADDRESS: pd:8686 + HG_STORE_GRPC_HOST: store2 + HG_STORE_GRPC_PORT: "8500" + HG_STORE_REST_PORT: "8520" + HG_STORE_RAFT_ADDRESS: store2:8510 + HG_STORE_DATA_PATH: /hugegraph-store/storage + HG_CLOUD_STORAGE_ENABLED: "true" + HG_CLOUD_STORAGE_BUCKET: "hugegraph-store2" + HG_CLOUD_STORAGE_ENDPOINT: "http://minio:9000" + HG_CLOUD_STORAGE_REGION: "us-east-1" + HG_CLOUD_STORAGE_ACCESS_KEY: "minioadmin" + HG_CLOUD_STORAGE_SECRET_KEY: "minioadmin" + HG_CLOUD_STORAGE_PATH_STYLE: "true" + HG_CLOUD_STORAGE_SYNC_INTERVAL_SECONDS: "30" + HG_CLOUD_STORAGE_SYNCHRONOUS_SST_UPLOAD_MODE: "true" + ports: ["8522:8520"] + volumes: [hg-store2-data:/hugegraph-store/storage] + networks: [hg-net] + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8520/v1/health >/dev/null || exit 1"] + interval: 10s + timeout: 10s + retries: 40 + start_period: 60s + server: + image: hugegraph/server:1.7.0 + container_name: cloud-storage-server + hostname: server + depends_on: + store0: + condition: service_healthy + store1: + condition: service_healthy + store2: + condition: service_healthy + environment: + STORE_REST: store0:8520 + ports: ["8080:8080"] + volumes: + - ${SERVER_GRAPH_CONF}:/hugegraph-server/conf/graphs/hugegraph.properties:ro + networks: [hg-net] + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8080/versions >/dev/null || exit 1"] + interval: 10s + timeout: 5s + retries: 40 + start_period: 60s +networks: + hg-net: + driver: bridge +volumes: + hg-minio-data: + hg-pd-data: + hg-store0-data: + hg-store1-data: + hg-store2-data: +YAML +prepare_artifacts +log "building local cloud-storage images..." +docker compose -f "$COMPOSE_FILE" build --no-cache 2>&1 | grep -E "^(Building|FINISHED|\[|Successfully|Error)" || true +log "starting minio and pd first..." +docker compose -f "$COMPOSE_FILE" up -d minio pd +wait_svc "minio" 120 +wait_svc "pd" 180 +NETWORK="${COMPOSE_PROJECT_NAME}_hg-net" +log "creating MinIO buckets before store startup..." +ensure_minio_buckets "$NETWORK" +log "starting stores and server..." +docker compose -f "$COMPOSE_FILE" up -d store0 store1 store2 server +log "waiting for stores..." +wait_svc "store0" 180 +wait_svc "store1" 180 +wait_svc "store2" 180 +wait_svc "server" 180 +log "waiting for graph backend..." +wait_http "$GRAPH_API_BASE/graph/vertices" 60 +log "✓ SUCCESS: Cloud storage infrastructure ready" diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index aa0736a38b..31a8be72d9 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -56,7 +56,7 @@ services: store: build: context: .. - dockerfile: hugegraph-store/Dockerfile + dockerfile: ${HG_STORE_DOCKERFILE:-hugegraph-store/Dockerfile} container_name: hg-store hostname: store restart: unless-stopped diff --git a/hugegraph-store/docs/pluggable-cloud-storage-architecture.md b/hugegraph-store/docs/pluggable-cloud-storage-architecture.md new file mode 100644 index 0000000000..e0a20d14d6 --- /dev/null +++ b/hugegraph-store/docs/pluggable-cloud-storage-architecture.md @@ -0,0 +1,574 @@ +# Pluggable Cloud Storage Architecture (HStore) + +This document explains: + +- The existing HStore workflow (without cloud storage) +- The new pluggable cloud storage workflow (cloud-backed SST lifecycle) +- Write path and read path behavior +- Failure handling and operational recovery scenarios + +## Overview + +HStore keeps partition data in local RocksDB on each Store node. The pluggable cloud-storage +feature extends this by mirroring SST file lifecycle events (create/delete) to an external +object store and hydrating missing files back when needed (startup and read-miss). + +- Existing path: local RocksDB is the primary online read/write path. +- New extension: cloud object storage acts as a pluggable SST mirror and recovery source. +- Provider model: runtime-selected SPI provider via `CloudStorageProviderFactory`. + +## Architecture Diagram + +```text ++---------------------------+ +| Client Layer | +| Gremlin/REST/Cypher | ++-------------|-------------+ + v ++---------------------------+ +----------------+ +| HugeGraph Server |-->| PD Cluster | ++-------------|-------------+ +----------------+ + v ++----------------------------------------------+ +| Store Cluster (Raft replication) | +| | +| +-----------------------------------------+ | +| | WAL + MemTable -> Local RocksDB SST | | +| +--------------------+--------------------+ | +| | | +| v | +| +-----------------------------------------+ | +| | Pluggable Cloud Storage Workflow (NEW) | | +| | CloudStorageEventListener (NEW) | | +| | -> CloudStorageProviderFactory (NEW) | | +| | -> CloudStorageProvider (NEW) | | +| +-----------------------------------------+ | ++----------------------|-----------------------+ + v + +-------------------------------+ + | Cloud Storage (NEW) | + | +-------------+ +----+ +----+ | + | |S3 compatible| |ADLS| |GCP | | + | +-------------+ +----+ +----+ | + +-------------------------------+ + +``` + +- `Client Layer`: External clients issuing Gremlin/REST/Cypher read-write requests. +- `HugeGraph Server`: API/query layer that routes graph requests to PD and Store. +- `PD Cluster`: Placement/metadata control plane (partition mapping, leader scheduling). +- `Store Cluster`: Raft-based data plane where writes are replicated and persisted. +- `WAL + MemTable -> Local RocksDB SST`: Local durability and compaction pipeline in each Store node. +- `Pluggable Cloud Storage Workflow (NEW)`: SST lifecycle hook path for upload/delete/download/list. +- `CloudStorageEventListener (NEW)`: Captures RocksDB file events and triggers cloud operations. +- `CloudStorageProviderFactory (NEW)`: SPI loader that selects and initializes the active provider. +- `CloudStorageProvider (NEW)`: Provider implementation (`s3`, etc.) used for object operations. +- `Cloud Storage (NEW)`: Remote object backend options (S3 compatible, ADSL, GCP). + + +## New Configuration Options + +The pluggable cloud-storage behavior is controlled from `application.yml` under +`cloud.storage`. + +| Configuration | Default | Description | +|--------------------------------------------|----------|---------------------------------------------------------------------------------------------------------------------| +| `cloud.storage.enabled` | `false` | Enables/disables cloud storage integration. | +| `cloud.storage.provider` | `s3` | Active provider name. Must match `CloudStorageProvider#providerName()`. | +| `cloud.storage.bucket` | _(none)_ | Target object storage bucket/container. Required when enabled. | +| `cloud.storage.region` | _(none)_ | Provider region (for example `us-east-1`). | +| `cloud.storage.endpoint` | _(none)_ | Optional custom endpoint for S3-compatible stores (MinIO/Ceph). | +| `cloud.storage.access-key` | _(none)_ | Access key / access ID credential. | +| `cloud.storage.secret-key` | _(none)_ | Secret key credential. | +| `cloud.storage.path-prefix` | `hugegraph` | Prefix prepended to all remote object keys. | +| `cloud.storage.startup-hydration-enabled` | `true` | Downloads missing remote files on DB opening before normal serving. | +| `cloud.storage.read-miss-guard-window-ms` | `3000` | Guard window to throttle repeated read-miss hydration attempts per db/table. Values `<= 0` disable throttling. | +| `cloud.storage.extra-properties` | `{}` | Provider-specific key/value map passed through during provider initialization. | + +Notes: + +- Keep `bucket` and `path-prefix` stable across restarts to preserve object-key continuity. +- If `provider` is not found on classpath, initialization fails fast in `CloudStorageProviderFactory`. + +## 3) Write Path + +```text +WRITE PATH (ANSI) + +Client + | + | 1) Write request + v +HugeGraph Server + | + | 2) Route to partition leader + v +Store Node (RocksDB) + | + | 3) WAL append + MemTable update + | 4) Flush/compaction creates *.sst + v +CloudStorageEventListener + | + | 5) onTableFileCreated(db, cf, file) + | 6) uploadFile(localPath, remoteKey) + v +CloudStorageProvider + | + | 7) PUT object + v +Object Storage + | + | 8) ACK + v +CloudStorageProvider -> CloudStorageEventListener (success) +``` + +### Write-path notes + +- On DB creation (`onDBCreated`), existing local SST files are scanned and uploaded if missing in cloud. +- An async flush is triggered so WAL-recovered/in-memory data materializes into SST and gets uploaded. +- Remote object key is derived from local path relative to Store data root. + +## 4) Read Path + +Two hydration modes exist: + +1. **Startup pre-hydration (`onDBOpening`)**: download missing remote files before serving. +2. **Read-miss hydration (`onReadMiss`)**: if local read misses, fetch missing SST from cloud, ingest, and retry. + +```text +READ PATH (ANSI) + +Read request + | + | 1) get(key) + v +RocksDB + | + | 2) miss -> onReadMiss(session, table, key) + v +CloudStorageEventListener + | + | 3) listFiles(dbPrefix) + v +CloudStorageProvider + | + | 4) LIST objects + v +Object Storage + | + | 5) return key list + v +CloudStorageProvider + | + | 6) for each missing *.sst: + | downloadFile(remoteKey, localPath) + | GET object -> receive object bytes + v +CloudStorageEventListener + | + | 7) ingestSstFile(downloaded) + v +RocksDB + | + | 8) retry get(key) + v +Read request result +``` + +### Read-path notes + +- A guard window (`read-miss-guard-window-ms`) throttles repeated hydration attempts for the same db/table pair. +- Only missing local SST files are downloaded. +- Non-SST objects are ignored for read-miss ingestion. + +## 5) Failure Handling + +### Upload failures (write-critical) + +- `onTableFileCreated` upload failure throws `IllegalStateException` (fail-fast). +- This surfaces cloud-sync write risk immediately instead of silently diverging local/cloud state. + +### Delete failures (non-fatal) + +- `onTableFileDeleted` delete failure is logged and processing continues. +- Impact is stale/orphaned cloud objects, not immediate read/write unavailability. + +### Hydration failures + +- Pre-hydration/list/download failures throw `IllegalStateException` and stop the flow for that DB open attempt. +- Read-miss hydration failures are logged and return `false`; caller falls back to original miss behavior. + +### Provider lifecycle / config failures + +- Unknown provider name or missing plugin JAR fails initialization in `CloudStorageProviderFactory`. +- Provider switching/re-init is handled with close-and-reinitialize semantics. + +## 6) Recovery Scenarios + +Failure-mode and RPO-oriented recovery summary: + +| Failure scenario | Data loss? | RPO | Recovery mechanism | Mitigation | +|---------------------------------------------|-------------------------------------------|--------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------| +| Single Store crash | No | 0s | 2/3 Raft quorum survives. Leader re-election continues service. In-flight (not SST-flushed) data is recovered from Raft logs; flushed data remains available via local/cloud SSTs. | Keep 3+ replicas across zones, alert on replica loss, and use durable PV-backed store nodes. | +| 2 of 3 Stores crash | No (service stalls until quorum restored) | 0s | Surviving replica Raft log bootstraps recovering nodes after restart. In-flight data is recovered from Raft log replay. | Enforce anti-affinity and failure-domain isolation to prevent correlated failures. | +| All Stores crash (disks intact) | No | 0s | Local Raft logs replay on boot and recover non-flushed in-flight data. Local SSTs reopen and cloud sync resumes for pending uploads. | Ensure orchestrator auto-restart, fast PV reattach, and regular restart drills. | +| Catastrophic loss: all Store disks destroyed | Yes | Seconds to minutes (depends on SST sync mode/interval) | Raft logs and local SSTs are lost. Nodes recover from last completed cloud SST sync; data after that sync is unrecoverable. | Use durable disks, scheduled volume snapshots/backups, and synchronous SST upload mode for tighter RPO. | + +Notes: + +- In-flight data (not yet flushed to SST) is recovered from Raft logs when quorum-replicated. +- Cloud storage recovery primarily protects flushed SST state and disaster cases involving local disk loss. +- Synchronous SST upload reduces catastrophic-loss RPO compared with periodic upload mode. + +## 7) Operational Notes + +- Prefer stable `path-prefix` and bucket naming; changing them affects object lookup continuity. +- Keep plugin JAR versions aligned with Store version to avoid SPI/API mismatch. +- Track logs around upload, hydration, and provider init to detect divergence early. +- For DR drills, test both node-level restart and full-cluster restart with cloud hydration enabled. + +## Plugin Development: Adding Custom Cloud Storage Providers + +This section guides developers on building and deploying new cloud storage provider plugins (e.g., Azure Blob Storage, ADLS, GCP). + +### Plugin Architecture Overview + +HugeGraph uses a **Service Provider Interface (SPI)** pattern to discover and load cloud storage providers at runtime: + +1. **CloudStorageProvider interface**: Located in `hugegraph-store/hg-store-common`, defines the contract all providers must implement. +2. **ServiceLoader discovery**: Store node uses `java.util.ServiceLoader` to find all `CloudStorageProvider` implementations on the classpath. +3. **SPI selection**: At Store startup, `CloudStorageProviderFactory` loads the provider specified in `cloud.storage.provider` config. +4. **Plugin packaging/runtime classpath**: Provider implementations are packaged as separate JAR modules and must be available on Store runtime classpath. + +### CloudStorageProvider Interface + +All custom providers must implement: + +```java +public interface CloudStorageProvider { + + /** + * Unique name for this provider (e.g., "s3", "azure", "adls"). + * Must match the `cloud.storage.provider` config value to be activated. + */ + String providerName(); + + /** + * Initialize the provider with configuration. + * Called once at Store startup. Throw exception if init fails. + */ + void init(CloudStorageConfig config) throws IOException; + + /** + * Upload a local file to cloud storage with the given remote key. + */ + void uploadFile(String localPath, String remoteKey) throws IOException; + + /** + * Download a remote file from cloud storage to the local path. + */ + void downloadFile(String remoteKey, String localPath) throws IOException; + + /** + * Delete a remote file from cloud storage. + */ + void deleteFile(String remoteKey) throws IOException; + + /** + * Check if a remote file exists. + */ + boolean fileExists(String remoteKey) throws IOException; + + /** + * List all remote files under the given directory prefix. + * Returns list of keys (with prefix stripped). + */ + List listFiles(String remoteDirPrefix) throws IOException; + + /** + * Close and release all provider resources. + */ + void close() throws IOException; +} +``` + +**Location:** `hugegraph-store/hg-store-common/src/main/java/org/apache/hugegraph/store/cloud/CloudStorageProvider.java` + +### Module Structure for a New Provider + +``` +hugegraph-store/ +├── hg-store-cloud-newprovider/ # New provider module +│ ├── pom.xml # Maven module definition +│ ├── src/main/java/org/apache/hugegraph/store/cloud/newprovider/ +│ │ └── NewProviderCloudStorageProvider.java # Implementation +│ ├── src/main/resources/META-INF/services/ +│ │ └── org.apache.hugegraph.store.cloud.CloudStorageProvider +│ └── src/test/java/... # Unit tests +``` + +Recommendation: + +- For providers maintained in this repository, prefer adding them as submodules under `hugegraph-store/` (same model as `hg-store-cloud-s3`). +- External providers are also supported if their jars are placed on runtime classpath. + +### Step 1: Create Module POM + +File: `hugegraph-store/hg-store-cloud-newprovider/pom.xml` + +```xml + + + 4.0.0 + + + org.apache.hugegraph + hugegraph-store + ${revision} + ../pom.xml + + + hg-store-cloud-newprovider + HugeGraph Store Cloud NewProvider + NewProvider Storage plugin for HugeGraph Store cloud storage integration + + + + + org.apache.hugegraph + hg-store-common + ${revision} + + + + + NewProvider + NewProvider + 1.0 + + + + +``` + +### Step 2: Implement the Provider + +File: `hugegraph-store/hg-store-cloud-newprovider/src/main/java/org/apache/hugegraph/store/cloud/newprovider/NewProviderCloudStorageProvider.java` + +```java +package org.apache.hugegraph.store.cloud.newprovider; + +import org.apache.hugegraph.store.cloud.CloudStorageConfig; +import org.apache.hugegraph.store.cloud.CloudStorageProvider; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +@SuppressWarnings("RedundantThrows") +@Slf4j +public class NewProviderCloudStorageProvider implements CloudStorageProvider { + + public static final String PROVIDER_NAME = "newprovider"; + + private Object providerClient; + private String pathPrefix; + + @Override + public String providerName() { + return PROVIDER_NAME; + } + + @Override + public void init(CloudStorageConfig config) throws IOException { + //Steps to initialize the NewProvider client using config parameters + } + + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + //Steps to upload file to NewProvider cloud storage + } + + @Override + public void downloadFile(String remoteKey, String localPath) throws IOException { + //Steps to download file from NewProvider cloud storage + } + + @Override + public void deleteFile(String remoteKey) throws IOException { + //Steps to delete file from NewProvider cloud storage + } + + @Override + public boolean fileExists(String remoteKey) throws IOException { + //Steps to check if file exists in NewProvider cloud storage + return false; + } + + @Override + public List listFiles(String remoteDirPrefix) throws IOException { + //Steps to list files in NewProvider cloud storage + return new ArrayList<>(); + } + + @Override + public void close() throws IOException { + //Steps to close and cleanup NewProvider client resources + } +} +``` + +### Step 3: Register via SPI + +File: `hugegraph-store/hg-store-cloud-newprovider/src/main/resources/META-INF/services/org.apache.hugegraph.store.cloud.CloudStorageProvider` + +``` +org.apache.hugegraph.store.cloud.newprovider.NewProviderCloudStorageProvider +``` + +This file tells Java's `ServiceLoader` to discover this provider at runtime. + +### Step 4: Build and Test + +```bash +# Build the provider module +cd hugegraph-store +mvn clean package -pl hg-store-cloud-newprovider -am -DskipTests + +# Find the generated JAR +find . -name "hg-store-cloud-newprovider-*.jar" +# Output: hugegraph-store/hg-store-cloud-newprovider/target/hg-store-cloud-newprovider-1.0.jar +``` + +### Step 5: Package for Runtime Classpath + +Use one of these runtime models: + +**Option A (recommended for in-repo providers): add as Store submodule dependency** + +1. Add module `hg-store-cloud-newprovider` under `hugegraph-store/`. +2. Add dependency from `hg-store-node` to `hg-store-cloud-newprovider`. +3. Rebuild distribution so provider + transitive dependencies are packaged with Store runtime. + +**Option B (external provider): provide jars on runtime classpath** + +- Supply `hg-store-cloud-newprovider-*.jar` **and** all required dependency jars, or +- supply a single shaded/uber provider jar that already contains transitive dependencies. + +Notes: + +- Simply copying a provider jar without its runtime dependencies can cause `ClassNotFoundException` during SPI loading. +- The exact classpath injection method depends on your launch model (custom startup script, container image, or JVM args). + +### Step 6: Configure and Activate + +In Store `application.yml`: + +```yaml +cloud: + storage: + enabled: true + provider: newprovider # Must match NewProviderCloudStorageProvider.providerName() + bucket: my-container + region: myaccount # Azure account name (using region field) + endpoint: https://myaccount.blob.core.windows.net + access-key: ${NEWPROVIDER_KEY} +``` + +Or via Docker env: + +```bash +HG_CLOUD_STORAGE_ENABLED=true \ +HG_CLOUD_STORAGE_PROVIDER=newprovider \ +HG_CLOUD_STORAGE_BUCKET=my-container \ +HG_CLOUD_STORAGE_REGION=myaccount \ +HG_CLOUD_STORAGE_ENDPOINT=https://myaccount.blob.core.windows.net \ +HG_CLOUD_STORAGE_ACCESS_KEY=${NEWPROVIDER_KEY} \ +docker run hugegraph-store:latest +``` + +### Testing Your Provider + +Add unit tests in `hg-store-cloud-newprovider/src/test/`: + +```java +@Test +public void testInit() { + // Test provider initialization with mock config +} + +@Test +public void testUploadDownload() { + // Test upload and download of a sample file +} + +@Test +public void testDeleteAndList() { + // Test delete and list operations +} + +@Test +public void testFileExists() { + // Test file existence check +} + +@Test +public void testClose() { + // Test provider close and resource cleanup +} +``` + +### Troubleshooting Provider Discovery + +If your provider isn't loaded: + +1. **Check SPI registration file exists:** + ```bash + jar tf hg-store-cloud-newprovider-1.0.jar | grep "META-INF/services" + ``` + Should show: `META-INF/services/org.apache.hugegraph.store.cloud.CloudStorageProvider` + +2. **Check provider name matches config:** + ```bash + jar xf hg-store-cloud-newprovider-1.0.jar META-INF/services/org.apache.hugegraph.store.cloud.CloudStorageProvider + cat META-INF/services/org.apache.hugegraph.store.cloud.CloudStorageProvider + ``` + +3. **Enable debug logging:** + ```yaml + logging: + level: + org.apache.hugegraph.store.cloud: DEBUG + org.apache.hugegraph.store.node.cloud: DEBUG + ``` + +4. **Verify provider jars are on classpath:** + ```bash + # Check Store startup logs for provider discovery / class-loading errors + docker logs hugegraph-store | grep -i "ServiceLoader\|provider" + ``` + +### Key Implementation Tips + +1. **Thread safety**: Ensure provider is thread-safe for concurrent upload/download/delete operations. +2. **Connection pooling**: Reuse client connections; initialize once in `init()`, close in `close()`. +3. **Path normalization**: Always use `pathPrefix` correctly (see S3 example for reference). +4. **Error handling**: Throw `IOException` for operational issues; let Store handle retries/logging. +5. **Logging**: Use SLF4J (via `@Slf4j`) for consistent log levels with Store. +6. **Configuration validation**: Validate all required fields in `init()` and fail fast. + +### Contributing Your Provider + +To upstream your new provider (e.g., `hg-store-cloud-newprovider`): + +1. Create PR to `hugegraph-store/` with new provider module +2. Add tests in `hg-store-test/` that validate provider behavior +3. Update `docker/cloud-storage/` examples with new provider setup steps +4. Update `install-dist/` license/notice if adding third-party dependencies diff --git a/hugegraph-store/hg-store-cloud-s3/pom.xml b/hugegraph-store/hg-store-cloud-s3/pom.xml new file mode 100644 index 0000000000..566a246ebd --- /dev/null +++ b/hugegraph-store/hg-store-cloud-s3/pom.xml @@ -0,0 +1,82 @@ + + + + + 4.0.0 + + + org.apache.hugegraph + hugegraph-store + ${revision} + ../pom.xml + + + hg-store-cloud-s3 + + + S3 cloud storage provider plugin for HugeGraph Store. + Add this JAR to the classpath and set cloud.storage.provider=s3 to enable S3 offload. + + + + 2.33.8 + + + + + + org.apache.hugegraph + hg-store-common + + + + + org.rocksdb + rocksdbjni + 7.7.3 + runtime + + + + + software.amazon.awssdk + s3 + ${aws.sdk.version} + + + + + software.amazon.awssdk + url-connection-client + ${aws.sdk.version} + + + + org.projectlombok + lombok + provided + + + diff --git a/hugegraph-store/hg-store-cloud-s3/src/main/java/org/apache/hugegraph/store/cloud/s3/S3CloudStorageProvider.java b/hugegraph-store/hg-store-cloud-s3/src/main/java/org/apache/hugegraph/store/cloud/s3/S3CloudStorageProvider.java new file mode 100644 index 0000000000..1b9ef765e8 --- /dev/null +++ b/hugegraph-store/hg-store-cloud-s3/src/main/java/org/apache/hugegraph/store/cloud/s3/S3CloudStorageProvider.java @@ -0,0 +1,250 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.cloud.s3; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hugegraph.store.cloud.CloudStorageConfig; +import org.apache.hugegraph.store.cloud.CloudStorageProvider; + +import lombok.extern.slf4j.Slf4j; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Object; + +/** + * Amazon S3 (and S3-compatible) implementation of {@link CloudStorageProvider}. + * + *

Activation

+ * Place {@code hg-store-cloud-s3-*.jar} on the classpath and configure: + * 
+ * cloud:
+ *   storage:
+ *     enabled: true
+ *     provider: s3
+ *     bucket:  my-bucket
+ *     region:  us-east-1
+ *
+ * + *

Credentials

+ *
    + *
  • If {@code access-key} / {@code secret-key} are set, they are used directly.
    • + *
  • Otherwise the standard AWS Default Credentials chain is followed + * (env vars, instance profile, ~/.aws/credentials, etc.).
    • + *
+ * + *

S3-compatible endpoints (MinIO, Ceph, etc.)

+ * Set {@code cloud.storage.endpoint} to the custom HTTP/HTTPS endpoint URL. + */ +@Slf4j +public class S3CloudStorageProvider implements CloudStorageProvider { + + /** Provider name as referenced in {@link CloudStorageConfig#getProvider()}. */ + public static final String PROVIDER_NAME = "s3"; + + private S3Client s3Client; + private String bucket; + private String pathPrefix; + + // ----------------------------------------------------------------------- + // CloudStorageProvider + // ----------------------------------------------------------------------- + + @Override + public String providerName() { + return PROVIDER_NAME; + } + + @Override + public void init(CloudStorageConfig config) { + this.bucket = config.getBucket(); + this.pathPrefix = config.getPathPrefix(); + + S3ClientBuilder builder = S3Client.builder(); + + // Credentials + String ak = config.getAccessKey(); + String sk = config.getSecretKey(); + if (ak != null && !ak.isEmpty() && sk != null && !sk.isEmpty()) { + builder.credentialsProvider( + StaticCredentialsProvider.create(AwsBasicCredentials.create(ak, sk))); + } else { + builder.credentialsProvider(DefaultCredentialsProvider.create()); + } + + // Region + String region = config.getRegion(); + if (region != null && !region.isEmpty()) { + builder.region(Region.of(region)); + } + + // Custom endpoint (MinIO, Ceph, LocalStack …) + String endpoint = config.getEndpoint(); + if (endpoint != null && !endpoint.isEmpty()) { + builder.endpointOverride(URI.create(endpoint)); + // Path-style required for most non-AWS S3 services + builder.serviceConfiguration( + software.amazon.awssdk.services.s3.S3Configuration.builder() + .pathStyleAccessEnabled(true) + .build()); + } + + this.s3Client = builder.build(); + log.info("S3CloudStorageProvider initialized: bucket='{}', region='{}', endpoint='{}'", + bucket, region, endpoint); + } + + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + String fullKey = buildKey(remoteKey); + try { + s3Client.putObject( + PutObjectRequest.builder().bucket(bucket).key(fullKey).build(), + Paths.get(localPath)); + log.debug("S3 upload: {} -> s3://{}/{}", localPath, bucket, fullKey); + } catch (SdkClientException e) { + throw new IOException( + "S3 upload failed for local='" + localPath + "' key='" + fullKey + "'", e); + } + } + + @Override + public void deleteFile(String remoteKey) throws IOException { + String fullKey = buildKey(remoteKey); + try { + s3Client.deleteObject( + DeleteObjectRequest.builder().bucket(bucket).key(fullKey).build()); + log.debug("S3 delete: s3://{}/{}", bucket, fullKey); + } catch (SdkClientException e) { + throw new IOException("S3 delete failed for key='" + fullKey + "'", e); + } + } + + @Override + public boolean fileExists(String remoteKey) throws IOException { + String fullKey = buildKey(remoteKey); + try { + s3Client.headObject(HeadObjectRequest.builder().bucket(bucket).key(fullKey).build()); + return true; + } catch (NoSuchKeyException e) { + return false; + } catch (SdkClientException e) { + throw new IOException("S3 headObject failed for key='" + fullKey + "'", e); + } + } + + @Override + public List listFiles(String remoteDirPrefix) throws IOException { + String fullPrefix = buildKey(remoteDirPrefix == null ? "" : remoteDirPrefix); + List keys = new ArrayList<>(); + try { + String token = null; + do { + ListObjectsV2Request.Builder req = + ListObjectsV2Request.builder().bucket(bucket).prefix(fullPrefix); + if (token != null) { + req.continuationToken(token); + } + ListObjectsV2Response resp = s3Client.listObjectsV2(req.build()); + for (S3Object obj : resp.contents()) { + String key = obj.key(); + if (key == null || key.endsWith("/")) { + continue; + } + keys.add(stripPathPrefix(key)); + } + token = resp.nextContinuationToken(); + } while (token != null && !token.isEmpty()); + return keys; + } catch (SdkClientException e) { + throw new IOException("S3 listObjects failed for prefix='" + fullPrefix + "'", e); + } + } + + @Override + public void downloadFile(String remoteKey, String localPath) throws IOException { + String fullKey = buildKey(remoteKey); + try { + s3Client.getObject( + GetObjectRequest.builder().bucket(bucket).key(fullKey).build(), + Paths.get(localPath)); + log.debug("S3 download: s3://{}/{} -> {}", bucket, fullKey, localPath); + } catch (SdkClientException e) { + throw new IOException( + "S3 download failed for key='" + fullKey + "' local='" + localPath + "'", e); + } + } + + @Override + public void close() throws IOException { + if (s3Client != null) { + s3Client.close(); + s3Client = null; + log.info("S3CloudStorageProvider closed"); + } + } + + // ----------------------------------------------------------------------- + // Internal helpers + // ----------------------------------------------------------------------- + + /** + * Prepends {@link #pathPrefix} to the supplied key, using "/" as separator. + * If the prefix is null or empty, the key is returned unchanged. + */ + private String buildKey(String key) { + if (pathPrefix == null || pathPrefix.isEmpty()) { + return key; + } + // Normalise leading slashes + String normalKey = key.startsWith("/") ? key.substring(1) : key; + return pathPrefix.endsWith("/") + ? pathPrefix + normalKey + : pathPrefix + "/" + normalKey; + } + + private String stripPathPrefix(String fullKey) { + if (fullKey == null) { + return ""; + } + if (pathPrefix == null || pathPrefix.isEmpty()) { + return fullKey.startsWith("/") ? fullKey.substring(1) : fullKey; + } + String normalizedPrefix = pathPrefix.endsWith("/") ? pathPrefix : pathPrefix + "/"; + if (fullKey.startsWith(normalizedPrefix)) { + return fullKey.substring(normalizedPrefix.length()); + } + return fullKey; + } +} diff --git a/hugegraph-store/hg-store-cloud-s3/src/main/resources/META-INF/services/org.apache.hugegraph.store.cloud.CloudStorageProvider b/hugegraph-store/hg-store-cloud-s3/src/main/resources/META-INF/services/org.apache.hugegraph.store.cloud.CloudStorageProvider new file mode 100644 index 0000000000..241d82992b --- /dev/null +++ b/hugegraph-store/hg-store-cloud-s3/src/main/resources/META-INF/services/org.apache.hugegraph.store.cloud.CloudStorageProvider @@ -0,0 +1,2 @@ +org.apache.hugegraph.store.cloud.s3.S3CloudStorageProvider + diff --git a/hugegraph-store/hg-store-common/src/main/java/org/apache/hugegraph/store/cloud/CloudStorageConfig.java b/hugegraph-store/hg-store-common/src/main/java/org/apache/hugegraph/store/cloud/CloudStorageConfig.java new file mode 100644 index 0000000000..da9121a942 --- /dev/null +++ b/hugegraph-store/hg-store-common/src/main/java/org/apache/hugegraph/store/cloud/CloudStorageConfig.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.cloud; + +import java.util.HashMap; +import java.util.Map; + +import lombok.Data; + +/** + * Configuration for pluggable cloud storage providers. + * + *

Mapped from application.yml under the {@code cloud.storage} prefix. Example: + * 

+ * cloud:
+ *   storage:
+ *     enabled: true
+ *     provider: s3
+ *     bucket: hugegraph-store
+ *     region: us-east-1
+ *     access-key: AKIAIOSFODNN7EXAMPLE
+ *     secret-key: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+ *     path-prefix: hugegraph/data
+ *
+ */ +@Data +public class CloudStorageConfig { + + /** + * Whether cloud storage is enabled. Defaults to false. + */ + private boolean enabled = false; + + /** + * Name of the provider to activate. Must match {@link CloudStorageProvider#providerName()}. + * The provider JAR must be on the classpath. Defaults to "s3". + */ + private String provider = "s3"; + + /** + * Cloud storage bucket / container name. + */ + private String bucket; + + /** + * Cloud region (e.g. "us-east-1"). + */ + private String region; + + /** + * Optional custom endpoint URL for S3-compatible stores (e.g. MinIO, Ceph). + * Leave empty to use the default AWS endpoint. + */ + private String endpoint; + + /** + * Access key / access ID credential. + */ + private String accessKey; + + /** + * Secret key / secret credential. + */ + private String secretKey; + + /** + * Key prefix prepended to every object stored in the bucket. + * Defaults to "hugegraph". + */ + private String pathPrefix = "hugegraph"; + + /** + * Whether startup pre-hydration (cloud -> local before DB open) is enabled. + */ + private boolean startupHydrationEnabled = true; + + /** + * Guard window in milliseconds for repeated read-miss hydration attempts on the + * same db/table pair. Values <= 0 disable the guard. + */ + private long readMissGuardWindowMs = 3000L; + + /** + * Provider-specific extra properties forwarded verbatim to the provider on init. + */ + private Map extraProperties = new HashMap<>(); +} diff --git a/hugegraph-store/hg-store-common/src/main/java/org/apache/hugegraph/store/cloud/CloudStorageProvider.java b/hugegraph-store/hg-store-common/src/main/java/org/apache/hugegraph/store/cloud/CloudStorageProvider.java new file mode 100644 index 0000000000..45a91e14dd --- /dev/null +++ b/hugegraph-store/hg-store-common/src/main/java/org/apache/hugegraph/store/cloud/CloudStorageProvider.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.cloud; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +/** + * SPI interface for pluggable cloud storage backends. + * + *

Implementations are discovered via {@link java.util.ServiceLoader}: + * each provider JAR must contain + * {@code META-INF/services/org.apache.hugegraph.store.cloud.CloudStorageProvider} + * listing the fully-qualified implementation class. + * + *

The active provider is selected at runtime through + * {@link CloudStorageConfig#getProvider()} and initialized once by + * {@link CloudStorageProviderFactory#initialize(CloudStorageConfig)}. + */ +public interface CloudStorageProvider extends Closeable { + + /** + * Returns the unique, lower-case name of this provider (e.g. {@code "s3"}, {@code "gcs"}). + * Must match the value set in {@link CloudStorageConfig#getProvider()}. + */ + String providerName(); + + /** + * Initializes the provider with the supplied configuration. + * Called once before any upload/download/delete operations. + * + * @param config cloud storage configuration + */ + void init(CloudStorageConfig config); + + /** + * Uploads a local file to cloud storage. + * + * @param localPath absolute path of the local source file + * @param remoteKey destination key / path inside the bucket (without prefix) + * @throws IOException on I/O or network failure + */ + void uploadFile(String localPath, String remoteKey) throws IOException; + + /** + * Deletes an object from cloud storage. + * + * @param remoteKey key / path of the object to delete (without prefix) + * @throws IOException on I/O or network failure + */ + void deleteFile(String remoteKey) throws IOException; + + /** + * Checks whether an object exists in cloud storage. + * + * @param remoteKey key / path to check (without prefix) + * @return {@code true} if the object exists + * @throws IOException on I/O or network failure + */ + boolean fileExists(String remoteKey) throws IOException; + + /** + * Lists object keys under a remote directory/prefix. + * + *

Default implementation returns an empty list so existing providers remain compatible. + * + * @param remoteDirPrefix directory/prefix inside bucket (without provider pathPrefix) + * @return object keys relative to provider root (without provider pathPrefix) + * @throws IOException on I/O or network failure + */ + default List listFiles(String remoteDirPrefix) throws IOException { + return Collections.emptyList(); + } + + /** + * Downloads an object from cloud storage to a local file. + * + * @param remoteKey destination key / path inside the bucket (without prefix) + * @param localPath absolute path of the local destination file + * @throws IOException on I/O or network failure + */ + void downloadFile(String remoteKey, String localPath) throws IOException; + + /** + * Releases resources held by the provider (e.g. HTTP clients, connections). + */ + @Override + void close() throws IOException; +} + diff --git a/hugegraph-store/hg-store-common/src/main/java/org/apache/hugegraph/store/cloud/CloudStorageProviderFactory.java b/hugegraph-store/hg-store-common/src/main/java/org/apache/hugegraph/store/cloud/CloudStorageProviderFactory.java new file mode 100644 index 0000000000..27644be9c4 --- /dev/null +++ b/hugegraph-store/hg-store-common/src/main/java/org/apache/hugegraph/store/cloud/CloudStorageProviderFactory.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.cloud; + +import java.io.IOException; +import java.util.Map; +import java.util.ServiceLoader; +import java.util.concurrent.ConcurrentHashMap; + +import lombok.Getter; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Factory for {@link CloudStorageProvider} instances. + * + *

Providers are discovered at class-loading time via {@link ServiceLoader}. + * Any JAR that includes + * {@code META-INF/services/org.apache.hugegraph.store.cloud.CloudStorageProvider} + * is automatically picked up when it is present on the classpath. + * + *

Usage: + * 

+ *   CloudStorageConfig cfg = ...; // populated from application.yml
+ *   CloudStorageProvider provider = CloudStorageProviderFactory.initialize(cfg);
+ *   // later:
+ *   CloudStorageProvider active = CloudStorageProviderFactory.getActiveProvider();
+ *
+ */ +public final class CloudStorageProviderFactory { + + private static final Logger log = LoggerFactory.getLogger(CloudStorageProviderFactory.class); + + /** All discovered providers keyed by {@link CloudStorageProvider#providerName()}. */ + private static final Map REGISTRY = new ConcurrentHashMap<>(); + + /** The currently active (initialized) provider; null when disabled or not yet initialized. + * -- GETTER -- + * Returns the currently active provider, or + * if cloud storage is + * disabled or + * has not yet been called. + */ + @Getter + private static volatile CloudStorageProvider activeProvider; + + static { + loadProviders(); + } + + private CloudStorageProviderFactory() { + } + + // ----------------------------------------------------------------------- + // Public API + // ----------------------------------------------------------------------- + + /** + * Initializes and activates the cloud storage provider described by {@code config}. + * + *

The method is idempotent: if called multiple times, the existing active + * provider is closed before a new one is initialized. + * + * @param config cloud storage configuration + * @return the initialized provider, or {@code null} when + * {@link CloudStorageConfig#isEnabled()} is {@code false} + * @throws IllegalArgumentException if no provider matching {@code config.getProvider()} + * was found on the classpath + */ + public static synchronized CloudStorageProvider initialize(CloudStorageConfig config) { + if (!config.isEnabled()) { + log.info("Cloud storage is disabled (cloud.storage.enabled=false)"); + return null; + } + + String name = config.getProvider(); + CloudStorageProvider provider = REGISTRY.get(name); + if (provider == null) { + throw new IllegalArgumentException( + "No cloud storage provider found for name '" + name + "'. " + + "Available providers: " + REGISTRY.keySet() + ". " + + "Make sure the provider JAR (e.g. hg-store-cloud-s3) is on the classpath."); + } + + // Close any previously active provider + if (activeProvider != null && activeProvider != provider) { + try { + activeProvider.close(); + } catch (IOException e) { + log.warn("Error closing previous cloud storage provider", e); + } + } + + provider.init(config); + activeProvider = provider; + log.info("Cloud storage provider '{}' initialized. bucket={}", name, config.getBucket()); + return provider; + } + + /** + * Shuts down and deregisters the active provider. + * Subsequent calls to {@link #getActiveProvider()} return {@code null}. + */ + public static synchronized void shutdown() { + if (activeProvider != null) { + try { + activeProvider.close(); + log.info("Cloud storage provider '{}' closed", activeProvider.providerName()); + } catch (IOException e) { + log.warn("Error closing cloud storage provider", e); + } + activeProvider = null; + } + } + + /** + * Resets the active provider to {@code null} without closing it. + * + *

For testing only. Use {@link #shutdown()} in production code. + */ + public static synchronized void reset() { + activeProvider = null; + } + + /** + * Directly injects an active provider, bypassing SPI discovery and {@link #initialize}. + * + *

For testing only. Allows tests to supply a stub/mock provider without + * placing a real JAR on the classpath. + * + * @param provider the provider instance to activate (may be {@code null} to clear) + */ + public static synchronized void setActiveProviderForTest(CloudStorageProvider provider) { + activeProvider = provider; + } + + // ----------------------------------------------------------------------- + // Internal helpers + // ----------------------------------------------------------------------- + + /** Scans the classpath for {@link CloudStorageProvider} implementations via SPI. */ + private static void loadProviders() { + ServiceLoader loader = + ServiceLoader.load(CloudStorageProvider.class, + CloudStorageProviderFactory.class.getClassLoader()); + for (CloudStorageProvider p : loader) { + String name = p.providerName(); + if (REGISTRY.containsKey(name)) { + log.warn("Duplicate cloud storage provider name '{}' – keeping first registration", + name); + } else { + REGISTRY.put(name, p); + log.info("Discovered cloud storage provider: '{}' ({})", + name, p.getClass().getName()); + } + } + if (REGISTRY.isEmpty()) { + log.debug("No cloud storage providers found on classpath; cloud storage is unavailable"); + } + } +} + diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/conf/application.yml b/hugegraph-store/hg-store-dist/src/assembly/static/conf/application.yml index 8e2b8d4f74..8c5957b0be 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/conf/application.yml +++ b/hugegraph-store/hg-store-dist/src/assembly/static/conf/application.yml @@ -62,3 +62,34 @@ logging: config: 'file:./conf/log4j2.xml' level: root: info + +# --------------------------------------------------------------------------- +# Cloud Storage (pluggable provider – disabled by default) +# --------------------------------------------------------------------------- +# To enable, set enabled=true and add the desired provider JAR to the +# classpath (e.g. hg-store-cloud-s3-*.jar for Amazon S3 / S3-compatible). +# +# cloud: +# storage: +# enabled: true +# provider: s3 # must match CloudStorageProvider.providerName() +# bucket: hugegraph-store # bucket / container name +# region: us-east-1 # AWS region (leave empty for custom endpoint) +# endpoint: # custom S3-compatible endpoint (MinIO, Ceph…) +# access-key: AKIAIOSFODNN7EXAMPLE # omit to use the AWS default credentials chain +# secret-key: wJalrXUtnFEMI/... # omit to use the AWS default credentials chain +# path-prefix: hugegraph # key prefix inside the bucket +# extra-properties: # provider-specific key-value pairs +# s3.forcePathStyle: "true" +cloud: + storage: + enabled: false + provider: s3 + bucket: + region: + endpoint: + access-key: + secret-key: + path-prefix: hugegraph + extra-properties: {} + diff --git a/hugegraph-store/hg-store-node/pom.xml b/hugegraph-store/hg-store-node/pom.xml index aff68d0db0..1c4296aec3 100644 --- a/hugegraph-store/hg-store-node/pom.xml +++ b/hugegraph-store/hg-store-node/pom.xml @@ -120,6 +120,10 @@ org.apache.hugegraph hg-store-core + + org.apache.hugegraph + hg-store-cloud-s3 + com.taobao.arthas diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java index 3f1624c087..2f7ace356a 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java @@ -17,11 +17,16 @@ package org.apache.hugegraph.store.node; +import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; import javax.annotation.PostConstruct; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.store.node.cloud.CloudStorageEventListener; +import org.apache.hugegraph.store.cloud.CloudStorageConfig; +import org.apache.hugegraph.store.cloud.CloudStorageProviderFactory; import org.apache.hugegraph.store.options.JobOptions; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -30,8 +35,10 @@ import org.springframework.stereotype.Component; import lombok.Data; +import lombok.extern.slf4j.Slf4j; @Data +@Slf4j @Component public class AppConfig { @@ -86,6 +93,9 @@ public class AppConfig { @Autowired private QueryPushDownConfig queryPushDownConfig; + @Autowired + private CloudStorageSpringConfig cloudStorageSpringConfig; + public String getRaftPath() { if (raftPath == null || raftPath.length() == 0) { return dataPath; @@ -116,6 +126,44 @@ public void init() { "0".equals(rocksdb.get("write_buffer_size"))) { rocksdb.put("write_buffer_size", Long.toString(totalMemory / 1000)); } + + // ---- Cloud storage initialization ---- + initCloudStorage(); + } + + /** + * Initialises the cloud storage provider (if enabled) and registers + * {@link CloudStorageEventListener} with {@link RocksDBFactory} so that + * SST file creation/deletion events are forwarded to cloud storage. + * + *

The resolved absolute data-path is passed to the listener so that + * S3 object keys are relative to the store's storage root rather than + * being full container-specific absolute paths. + */ + private void initCloudStorage() { + CloudStorageConfig cfg = cloudStorageSpringConfig.toCloudStorageConfig(); + if (!cfg.isEnabled()) { + log.info("Cloud storage disabled (cloud.storage.enabled=false)"); + return; + } + try { + CloudStorageProviderFactory.initialize(cfg); + String resolvedDataRoot = + Paths.get(dataPath).toAbsolutePath().normalize().toString(); + RocksDBFactory.getInstance().addRocksdbChangedListener( + new CloudStorageEventListener(resolvedDataRoot, + cfg.isStartupHydrationEnabled(), + cfg.getReadMissGuardWindowMs())); + log.info("Cloud storage provider '{}' registered with RocksDBFactory " + + "(dataRoot='{}', startupHydration={}, readMissHydration=true, " + + "readMissGuardWindowMs={})", + cfg.getProvider(), resolvedDataRoot, + cfg.isStartupHydrationEnabled(), + cfg.getReadMissGuardWindowMs()); + } catch (Exception e) { + log.error("Failed to initialize cloud storage provider '{}': {}", + cfg.getProvider(), e.getMessage(), e); + } } @Override @@ -312,6 +360,54 @@ public class RocksdbConfig { private Map rocksdb = new HashMap<>(); } + /** + * Spring {@link ConfigurationProperties} wrapper around {@link CloudStorageConfig}. + * + *

Mapped from the {@code cloud.storage.*} YAML namespace. Example: + * 

+     * cloud:
+     *   storage:
+     *     enabled: true
+     *     provider: s3
+     *     bucket:  my-bucket
+     *     region:  us-east-1
+     *
+ */ + @Data + @Configuration + @ConfigurationProperties(prefix = "cloud.storage") + public class CloudStorageSpringConfig { + + private boolean enabled = false; + private String provider = "s3"; + private String bucket; + private String region; + private String endpoint; + private String accessKey; + private String secretKey; + private String pathPrefix = "hugegraph"; + private boolean startupHydrationEnabled = true; + private long readMissGuardWindowMs = 3000L; + private Map extraProperties = new HashMap<>(); + + /** Converts this Spring-bound config into a plain {@link CloudStorageConfig} POJO. */ + public CloudStorageConfig toCloudStorageConfig() { + CloudStorageConfig cfg = new CloudStorageConfig(); + cfg.setEnabled(enabled); + cfg.setProvider(provider); + cfg.setBucket(bucket); + cfg.setRegion(region); + cfg.setEndpoint(endpoint); + cfg.setAccessKey(accessKey); + cfg.setSecretKey(secretKey); + cfg.setPathPrefix(pathPrefix); + cfg.setStartupHydrationEnabled(startupHydrationEnabled); + cfg.setReadMissGuardWindowMs(readMissGuardWindowMs); + cfg.setExtraProperties(extraProperties); + return cfg; + } + } + public JobOptions getJobOptions() { JobOptions jobOptions = new JobOptions(); jobOptions.setCore(jobConfig.getCore() == 0 ? cpus : jobConfig.getCore()); diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/cloud/CloudStorageEventListener.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/cloud/CloudStorageEventListener.java new file mode 100644 index 0000000000..fc253442fa --- /dev/null +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/cloud/CloudStorageEventListener.java @@ -0,0 +1,417 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node.cloud; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Stream; + +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory.RocksdbChangedListener; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.store.cloud.CloudStorageProvider; +import org.apache.hugegraph.store.cloud.CloudStorageProviderFactory; + +import lombok.extern.slf4j.Slf4j; + +/** + * {@link RocksdbChangedListener} that bridges RocksDB table-file lifecycle events + * to the active {@link CloudStorageProvider}. + * + *

When cloud storage is enabled: + *

    + *
  • {@link #onDBCreated} uploads any SST files that already exist in the DB directory + * (e.g. surviving from a previous run) and triggers an async MemTable flush so that + * WAL-recovered or recently-written data is also written to SST files.
    • + *
  • {@link #onTableFileCreated} uploads newly created SST files.
    • + *
  • {@link #onTableFileDeleted} removes the corresponding object from cloud storage.
    • + *
+ * + *

Remote key construction

+ * The remote key is derived by stripping the {@code dataRoot} prefix from the absolute + * local file path. This keeps the object layout clean and independent of the container + * filesystem layout: + * 
+ *   dataRoot  = /hugegraph-store/storage
+ *   filePath  = /hugegraph-store/storage/hgstore-metadata/000008.sst
+ *   remoteKey = hgstore-metadata/000008.sst
+ *   (with path-prefix "hugegraph") → hugegraph/hgstore-metadata/000008.sst
+ *
+ * + * This listener is registered with {@link RocksDBFactory} during application startup + * (see {@link org.apache.hugegraph.store.node.AppConfig}). + */ +@Slf4j +public class CloudStorageEventListener implements RocksdbChangedListener { + + /** Absolute, normalised path of the store's data root directory. */ + private final String dataRoot; + + private static final long DEFAULT_READ_MISS_GUARD_WINDOW_MS = 3000L; + + private final boolean startupHydrationEnabled; + private final long readMissGuardWindowMs; + private final Map readMissAttemptTs; + + /** + * @param dataRoot absolute path of the store's data directory + * (value of {@code app.data-path}, resolved to an absolute path). + */ + public CloudStorageEventListener(String dataRoot) { + this(dataRoot, true, DEFAULT_READ_MISS_GUARD_WINDOW_MS); + } + + public CloudStorageEventListener(String dataRoot, + boolean startupHydrationEnabled) { + this(dataRoot, startupHydrationEnabled, DEFAULT_READ_MISS_GUARD_WINDOW_MS); + } + + /** + * @param readMissGuardWindowMs guard window in ms for repeated read-miss hydration attempts + * for the same db/table pair (cloud.storage.read-miss-guard-window-ms) + */ + public CloudStorageEventListener(String dataRoot, + boolean startupHydrationEnabled, + long readMissGuardWindowMs) { + String normalised = Paths.get(dataRoot).toAbsolutePath().normalize().toString(); + // Strip trailing separator so substring arithmetic is consistent. + this.dataRoot = normalised.endsWith(File.separator) + ? normalised.substring(0, normalised.length() - 1) + : normalised; + this.startupHydrationEnabled = startupHydrationEnabled; + this.readMissGuardWindowMs = Math.max(0L, readMissGuardWindowMs); + this.readMissAttemptTs = new ConcurrentHashMap<>(); + } + + // ----------------------------------------------------------------------- + // RocksdbChangedListener + // ----------------------------------------------------------------------- + + @Override + public void onDBOpening(String dbName, String dbPath) { + if (!startupHydrationEnabled) { + return; + } + CloudStorageProvider provider = CloudStorageProviderFactory.getActiveProvider(); + if (provider == null) { + return; + } + preHydrateDbFiles(provider, dbName, dbPath); + } + + /** + * Called when a read returns null in RocksDB. We try to hydrate missing SST files from cloud, + * ingest them into the target CF, then caller retries get(). + */ + @Override + public boolean onReadMiss(RocksDBSession session, String table, byte[] key) { + if (!shouldAttemptReadMissHydration(session.getGraphName(), table)) { + return false; + } + CloudStorageProvider provider = CloudStorageProviderFactory.getActiveProvider(); + if (provider == null) { + return false; + } + List downloaded = downloadMissingSstFiles(provider, + session.getGraphName(), + session.getDbPath()); + if (downloaded.isEmpty()) { + return false; + } + try { + Map> sstByCf = new HashMap<>(); + sstByCf.put(table.getBytes(StandardCharsets.UTF_8), downloaded); + session.ingestSstFile(sstByCf); + log.info("Cloud read-miss hydration succeeded: db={}, table={}, files={}", + session.getGraphName(), table, downloaded.size()); + return true; + } catch (Exception e) { + log.warn("Cloud read-miss hydration failed: db={}, table={}, reason={}", + session.getGraphName(), table, e.getMessage()); + return false; + } + } + + /** + * Called when a new RocksDB instance is opened for the first time. + * + *

Uploads any SST files that already exist in {@code dbPath} (e.g. from a previous run) + * and then triggers a MemTable flush so that WAL-recovered data is also written to + * SST files and eventually forwarded here via {@link #onTableFileCreated}. + * + * @param dbName logical name of the graph / partition + * @param dbPath absolute path of the RocksDB directory + */ + @Override + public void onDBCreated(String dbName, String dbPath) { + CloudStorageProvider provider = CloudStorageProviderFactory.getActiveProvider(); + if (provider == null) { + return; + } + uploadExistingSstFiles(provider, dbName, dbPath); + flushDb(dbName); + } + + /** + * Uploads the newly created SST file to the active cloud storage provider. + * + * @param dbName RocksDB instance name (partition id) + * @param cfName column-family name + * @param filePath absolute local path of the new SST file + * @param fileSize file size in bytes (informational) + */ + @Override + public void onTableFileCreated(String dbName, String cfName, + String filePath, long fileSize) { + CloudStorageProvider provider = CloudStorageProviderFactory.getActiveProvider(); + if (provider == null) { + return; + } + String remoteKey = toRelativeKey(filePath); + try { + provider.uploadFile(filePath, remoteKey); + log.debug("Cloud upload success: db={}, cf={}, path={}, size={}", + dbName, cfName, filePath, fileSize); + } catch (Exception e) { + log.error("Cloud upload failed: db={}, cf={}, path={}", dbName, cfName, filePath, e); + throw new IllegalStateException( + String.format("Cloud upload failed for db=%s cf=%s path=%s", + dbName, cfName, filePath), e); + } + } + + /** + * Removes the deleted SST file from the active cloud storage provider. + * + * @param dbName RocksDB instance name (partition id) + * @param cfName column-family name + * @param filePath absolute local path of the deleted SST file + */ + @Override + public void onTableFileDeleted(String dbName, String cfName, String filePath) { + CloudStorageProvider provider = CloudStorageProviderFactory.getActiveProvider(); + if (provider == null) { + return; + } + String remoteKey = toRelativeKey(filePath); + try { + provider.deleteFile(remoteKey); + log.debug("Cloud delete success: db={}, cf={}, path={}", dbName, cfName, filePath); + } catch (Exception e) { + // Non-fatal: log and continue. + log.error("Cloud delete failed: db={}, cf={}, path={}", dbName, cfName, filePath, e); + } + } + + // ----------------------------------------------------------------------- + // Internal helpers + // ----------------------------------------------------------------------- + + /** + * Converts an absolute local file path to a remote key by stripping the data-root prefix. + * + * 

+     *   dataRoot = /hugegraph-store/storage
+     *   filePath = /hugegraph-store/storage/hgstore-metadata/000008.sst
+     *   result   = hgstore-metadata/000008.sst
+     *
+ * + * If {@code filePath} does not start with {@code dataRoot} the leading slash is simply + * stripped so the key is still valid (though possibly not ideallyformatted). + */ + String toRelativeKey(String filePath) { + if (filePath.startsWith(dataRoot)) { + String rel = filePath.substring(dataRoot.length()); + // Strip leading separator produced by the substring. + return rel.startsWith("/") || rel.startsWith(File.separator) + ? rel.substring(1) + : rel; + } + // Fallback: strip any leading slash so the key does not start with '/'. + return filePath.startsWith("/") ? filePath.substring(1) : filePath; + } + + /** + * Walks {@code dbPath} and uploads every {@code *.sst} file that is not already + * present in cloud storage. This handles restarts where SST files from a previous + * run were never uploaded (e.g. cloud storage was enabled after the last shutdown). + */ + private void uploadExistingSstFiles(CloudStorageProvider provider, String dbName, + String dbPath) { + Path root = Paths.get(dbPath); + if (!root.toFile().isDirectory()) { + return; + } + try (Stream paths = Files.walk(root)) { + paths.filter(p -> p.toString().endsWith(".sst")) + .forEach(p -> { + String localPath = p.toString(); + String remoteKey = toRelativeKey(localPath); + try { + if (!provider.fileExists(remoteKey)) { + provider.uploadFile(localPath, remoteKey); + log.info("Cloud initial-upload: {} -> {}", localPath, remoteKey); + } + } catch (IOException e) { + throw new IllegalStateException( + String.format("Cloud initial-upload failed for db=%s path=%s", + dbName, localPath), e); + } + }); + } catch (IOException e) { + throw new IllegalStateException( + String.format("Cloud initial-upload scan failed for db=%s dbPath=%s", + dbName, dbPath), e); + } + } + + private void preHydrateDbFiles(CloudStorageProvider provider, String dbName, String dbPath) { + Path root = Paths.get(dbPath); + try { + Files.createDirectories(root); + } catch (IOException e) { + throw new IllegalStateException( + String.format("Cloud pre-hydration mkdir failed for db=%s path=%s", + dbName, dbPath), e); + } + + String prefix = dbPrefix(dbPath); + List remoteFiles = listRemoteKeys(provider, prefix); + if (remoteFiles.isEmpty()) { + log.debug("Cloud pre-hydration skipped: no remote files for db={} prefix={}", + dbName, prefix); + return; + } + + int downloaded = 0; + for (String remoteKey : remoteFiles) { + Path localPath = resolveLocalPath(remoteKey); + if (Files.exists(localPath)) { + continue; + } + try { + Files.createDirectories(localPath.getParent()); + provider.downloadFile(remoteKey, localPath.toString()); + downloaded++; + } catch (IOException e) { + throw new IllegalStateException( + String.format("Cloud pre-hydration failed for db=%s key=%s", + dbName, remoteKey), e); + } + } + if (downloaded > 0) { + log.info("Cloud pre-hydration finished: db={}, downloadedFiles={}", dbName, downloaded); + } + } + + private List downloadMissingSstFiles(CloudStorageProvider provider, + String dbName, + String dbPath) { + String prefix = dbPrefix(dbPath); + List remoteFiles = listRemoteKeys(provider, prefix); + if (remoteFiles.isEmpty()) { + return List.of(); + } + + List downloaded = new ArrayList<>(); + for (String remoteKey : remoteFiles) { + if (!remoteKey.endsWith(".sst")) { + continue; + } + Path localPath = resolveLocalPath(remoteKey); + if (Files.exists(localPath)) { + continue; + } + try { + Files.createDirectories(localPath.getParent()); + provider.downloadFile(remoteKey, localPath.toString()); + downloaded.add(localPath.toString()); + } catch (IOException e) { + throw new IllegalStateException( + String.format("Cloud read-miss download failed for db=%s key=%s", + dbName, remoteKey), e); + } + } + return downloaded; + } + + private List listRemoteKeys(CloudStorageProvider provider, String prefix) { + try { + return provider.listFiles(prefix.endsWith("/") ? prefix : prefix + "/"); + } catch (IOException e) { + throw new IllegalStateException( + String.format("Cloud list failed for prefix=%s", prefix), e); + } + } + + private String dbPrefix(String dbPath) { + String relative = toRelativeKey(dbPath); + return relative.endsWith("/") ? relative.substring(0, relative.length() - 1) : relative; + } + + private Path resolveLocalPath(String remoteKey) { + Path root = Paths.get(this.dataRoot); + Path local = root.resolve(remoteKey).normalize(); + if (!local.startsWith(root)) { + throw new IllegalArgumentException("Invalid remote key outside data root: " + remoteKey); + } + return local; + } + + /** + * Triggers an asynchronous MemTable flush for the named DB via {@link RocksDBFactory}. + * This causes any in-memory data (including WAL-recovered entries) to be written to an + * SST file, which in turn fires {@link #onTableFileCreated} and uploads the file. + */ + private void flushDb(String dbName) { + try { + RocksDBFactory.getInstance().flushSession(dbName, false); + log.debug("Cloud storage: triggered async flush for db={}", dbName); + } catch (Exception e) { + log.warn("Cloud storage: flush failed for db={}: {}", dbName, e.getMessage()); + } + } + + private boolean shouldAttemptReadMissHydration(String dbName, String table) { + if (readMissGuardWindowMs <= 0) { + return true; + } + long now = System.currentTimeMillis(); + String guardKey = dbName + "::" + table; + Long prev = readMissAttemptTs.put(guardKey, now); + if (prev == null) { + return true; + } + long elapsed = now - prev; + if (elapsed >= readMissGuardWindowMs) { + return true; + } + log.debug("Skip read-miss hydration due to guard window: db={}, table={}, elapsedMs={}", + dbName, table, elapsed); + return false; + } +} diff --git a/hugegraph-store/hg-store-node/src/main/resources/application.yml b/hugegraph-store/hg-store-node/src/main/resources/application.yml index 0b65270608..5ce3ff9a3e 100644 --- a/hugegraph-store/hg-store-node/src/main/resources/application.yml +++ b/hugegraph-store/hg-store-node/src/main/resources/application.yml @@ -49,3 +49,17 @@ logging: config: classpath:log4j2-dev.xml level: root: info + +# Cloud storage (disabled by default; see hg-store-dist application.yml for full docs) +cloud: + storage: + enabled: false + provider: s3 + bucket: + region: + endpoint: + access-key: + secret-key: + path-prefix: hugegraph + extra-properties: {} + diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/cloud/CloudStorageEventListenerTest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/cloud/CloudStorageEventListenerTest.java new file mode 100644 index 0000000000..6ad075695e --- /dev/null +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/cloud/CloudStorageEventListenerTest.java @@ -0,0 +1,392 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node.cloud; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.store.cloud.CloudStorageProvider; +import org.apache.hugegraph.store.cloud.CloudStorageProviderFactory; +import org.apache.hugegraph.store.cloud.CloudStorageConfig; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Unit tests for {@link CloudStorageEventListener}. + * + *

Focuses on the relative-key computation ({@code toRelativeKey}) and + * the {@link CloudStorageEventListener#onTableFileCreated} / {@link + * CloudStorageEventListener#onTableFileDeleted} delegate calls to the + * active {@link CloudStorageProvider}. + */ +public class CloudStorageEventListenerTest { + + private static final String DATA_ROOT = "/hugegraph-store/storage"; + + private CloudStorageEventListener listener; + + @Before + public void setUp() { + listener = new CloudStorageEventListener(DATA_ROOT); + } + + @After + public void tearDown() { + // Reset active provider between tests + CloudStorageProviderFactory.reset(); + } + + // ----------------------------------------------------------------------- + // toRelativeKey + // ----------------------------------------------------------------------- + + @Test + public void toRelativeKey_stripsDataRootPrefix() { + String filePath = DATA_ROOT + "/hgstore-metadata/000008.sst"; + assertEquals("hgstore-metadata/000008.sst", listener.toRelativeKey(filePath)); + } + + @Test + public void toRelativeKey_stripsDataRootPrefixForPartitionDb() { + String filePath = DATA_ROOT + "/0/000042.sst"; + assertEquals("0/000042.sst", listener.toRelativeKey(filePath)); + } + + @Test + public void toRelativeKey_fallsBackToStripLeadingSlash_whenNotUnderDataRoot() { + String filePath = "/some/other/path/000001.sst"; + assertEquals("some/other/path/000001.sst", listener.toRelativeKey(filePath)); + } + + @Test + public void toRelativeKey_handlesDataRootWithTrailingSlash() { + CloudStorageEventListener l = + new CloudStorageEventListener(DATA_ROOT + File.separator); + assertEquals("hgstore-metadata/000008.sst", + l.toRelativeKey(DATA_ROOT + "/hgstore-metadata/000008.sst")); + } + + // ----------------------------------------------------------------------- + // onTableFileCreated / onTableFileDeleted – no active provider (no-op) + // ----------------------------------------------------------------------- + + @Test + public void onTableFileCreated_noActiveProvider_doesNotThrow() { + // No provider registered → method must be a silent no-op. + listener.onTableFileCreated("hgstore-metadata", "default", + DATA_ROOT + "/hgstore-metadata/000008.sst", 1234L); + } + + @Test + public void onTableFileDeleted_noActiveProvider_doesNotThrow() { + listener.onTableFileDeleted("hgstore-metadata", "default", + DATA_ROOT + "/hgstore-metadata/000008.sst"); + } + + // ----------------------------------------------------------------------- + // onTableFileCreated / onTableFileDeleted – with stub provider + // ----------------------------------------------------------------------- + + @Test + public void onTableFileCreated_delegatesToProvider_withRelativeKey() { + CapturingProvider provider = new CapturingProvider(); + CloudStorageProviderFactory.setActiveProviderForTest(provider); + + listener.onTableFileCreated("hgstore-metadata", "default", + DATA_ROOT + "/hgstore-metadata/000008.sst", 512L); + + assertEquals(1, provider.uploads.size()); + assertEquals(DATA_ROOT + "/hgstore-metadata/000008.sst", provider.uploads.get(0)[0]); + assertEquals("hgstore-metadata/000008.sst", provider.uploads.get(0)[1]); + } + + @Test + public void onTableFileCreated_uploadFailure_throwsRuntimeException() { + CloudStorageProviderFactory.setActiveProviderForTest(new FailingUploadProvider()); + + try { + listener.onTableFileCreated("hgstore-metadata", "default", + DATA_ROOT + "/hgstore-metadata/000008.sst", 512L); + fail("Expected upload failure to be rethrown"); + } catch (IllegalStateException e) { + assertTrue(e.getMessage().contains("Cloud upload failed")); + } + } + + @Test + public void onTableFileDeleted_delegatesToProvider_withRelativeKey() { + CapturingProvider provider = new CapturingProvider(); + CloudStorageProviderFactory.setActiveProviderForTest(provider); + + listener.onTableFileDeleted("hgstore-metadata", "default", + DATA_ROOT + "/hgstore-metadata/000008.sst"); + + assertEquals(1, provider.deletes.size()); + assertEquals("hgstore-metadata/000008.sst", provider.deletes.get(0)); + } + + // ----------------------------------------------------------------------- + // onDBCreated – uploads existing SST files from the DB directory + // ----------------------------------------------------------------------- + + @Test + public void onDBCreated_uploadsExistingSstFiles() throws Exception { + // Create a temporary directory that mimics a partition DB directory. + Path tmpRoot = Files.createTempDirectory("hgstore-test-storage"); + Path partitionDir = tmpRoot.resolve("0"); + Files.createDirectories(partitionDir); + Files.createFile(partitionDir.resolve("000001.sst")); + Files.createFile(partitionDir.resolve("000002.sst")); + + CloudStorageEventListener l = + new CloudStorageEventListener(tmpRoot.toString()); + CapturingProvider provider = new CapturingProvider(); + CloudStorageProviderFactory.setActiveProviderForTest(provider); + + try { + // Trigger onDBCreated; startup backfill should upload both SST files synchronously. + l.onDBCreated("0", partitionDir.toString()); + + assertEquals(2, provider.uploads.size()); + List remoteKeys = new ArrayList<>(); + for (String[] u : provider.uploads) { + remoteKeys.add(u[1]); + } + // Keys should be relative (e.g. "0/000001.sst") + for (String key : remoteKeys) { + assert !key.startsWith("/") : "Remote key must not start with '/': " + key; + assert key.endsWith(".sst") : "Remote key must end with .sst: " + key; + } + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onDBCreated_existingUploadFailure_throwsRuntimeException() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-storage"); + Path partitionDir = tmpRoot.resolve("0"); + Files.createDirectories(partitionDir); + Files.createFile(partitionDir.resolve("000001.sst")); + + CloudStorageEventListener l = + new CloudStorageEventListener(tmpRoot.toString()); + CloudStorageProviderFactory.setActiveProviderForTest(new FailingUploadProvider()); + + try { + try { + l.onDBCreated("0", partitionDir.toString()); + fail("Expected startup backfill failure to be rethrown"); + } catch (IllegalStateException e) { + assertTrue(e.getMessage().contains("Cloud initial-upload failed")); + } + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + // ----------------------------------------------------------------------- + // Helpers + // ----------------------------------------------------------------------- + + @SuppressWarnings("ResultOfMethodCallIgnored") + private void deleteRecursively(File f) { + if (f.isDirectory()) { + for (File child : Objects.requireNonNull(f.listFiles())) { + deleteRecursively(child); + } + } + f.delete(); + } + + /** + * Minimal {@link CloudStorageProvider} that records upload and delete calls. + */ + static class CapturingProvider implements CloudStorageProvider { + + final List uploads = new ArrayList<>(); + final List deletes = new ArrayList<>(); + final Map remoteFiles = new HashMap<>(); + int listFilesCalls = 0; + + @Override + public String providerName() { + return "capturing"; + } + + @Override + public void init(CloudStorageConfig config) { + } + + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + uploads.add(new String[]{localPath, remoteKey}); + } + + @SuppressWarnings("RedundantThrows") + @Override + public void deleteFile(String remoteKey) throws IOException { + deletes.add(remoteKey); + } + + @SuppressWarnings("RedundantThrows") + @Override + public boolean fileExists(String remoteKey) throws IOException { + return remoteFiles.containsKey(remoteKey); + } + + @SuppressWarnings("RedundantThrows") + @Override + public List listFiles(String remoteDirPrefix) throws IOException { + listFilesCalls++; + List result = new ArrayList<>(); + for (String key : remoteFiles.keySet()) { + if (key.startsWith(remoteDirPrefix)) { + result.add(key); + } + } + return result; + } + + @Override + public void downloadFile(String remoteKey, String localPath) throws IOException { + byte[] data = remoteFiles.get(remoteKey); + if (data == null) { + throw new IOException("remote key not found: " + remoteKey); + } + Path p = Paths.get(localPath); + Files.createDirectories(p.getParent()); + Files.write(p, data); + } + + @SuppressWarnings("RedundantThrows") + @Override + public void close() throws IOException { + } + + void putRemoteFile(String key, byte[] content) { + remoteFiles.put(key, content); + } + } + + static class FailingUploadProvider extends CapturingProvider { + + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + throw new IOException("simulated upload failure"); + } + } + + @Test + public void onDBOpening_downloadsMissingRemoteFiles() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-storage"); + Path partitionDir = tmpRoot.resolve("0"); + Files.createDirectories(partitionDir); + + CloudStorageEventListener l = + new CloudStorageEventListener(tmpRoot.toString(), true); + CapturingProvider provider = new CapturingProvider(); + provider.putRemoteFile("0/CURRENT", "MANIFEST-000001".getBytes()); + provider.putRemoteFile("0/MANIFEST-000001", "manifest-body".getBytes()); + provider.putRemoteFile("0/000001.sst", "sst-body".getBytes()); + CloudStorageProviderFactory.setActiveProviderForTest(provider); + + try { + l.onDBOpening("0", partitionDir.toString()); + assertTrue(Files.exists(partitionDir.resolve("CURRENT"))); + assertTrue(Files.exists(partitionDir.resolve("MANIFEST-000001"))); + assertTrue(Files.exists(partitionDir.resolve("000001.sst"))); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onReadMiss_downloadsSstAndRequestsIngest() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-storage"); + Path partitionDir = tmpRoot.resolve("0"); + Files.createDirectories(partitionDir); + + CloudStorageEventListener l = + new CloudStorageEventListener(tmpRoot.toString(), true); + CapturingProvider provider = new CapturingProvider(); + provider.putRemoteFile("0/000001.sst", "sst-body".getBytes()); + CloudStorageProviderFactory.setActiveProviderForTest(provider); + + RocksDBSession session = mock(RocksDBSession.class); + when(session.getGraphName()).thenReturn("0"); + when(session.getDbPath()).thenReturn(partitionDir.toString()); + + try { + boolean hydrated = l.onReadMiss(session, "default", "k".getBytes()); + assertTrue(hydrated); + verify(session, times(1)).ingestSstFile(anyMap()); + assertTrue(Files.exists(partitionDir.resolve("000001.sst"))); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onReadMiss_guardSkipsRepeatedAttemptsWithinWindow() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-storage"); + Path partitionDir = tmpRoot.resolve("0"); + Files.createDirectories(partitionDir); + + CloudStorageEventListener l = + new CloudStorageEventListener(tmpRoot.toString(), true, 60_000L); + CapturingProvider provider = new CapturingProvider(); + provider.putRemoteFile("0/000001.sst", "sst-body".getBytes()); + CloudStorageProviderFactory.setActiveProviderForTest(provider); + + RocksDBSession session = mock(RocksDBSession.class); + when(session.getGraphName()).thenReturn("0"); + when(session.getDbPath()).thenReturn(partitionDir.toString()); + + try { + boolean first = l.onReadMiss(session, "default", "k1".getBytes()); + boolean second = l.onReadMiss(session, "default", "k2".getBytes()); + assertTrue(first); + assertFalse(second); + verify(session, times(1)).ingestSstFile(anyMap()); + assertEquals(1, provider.listFilesCalls); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } +} diff --git a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBFactory.java b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBFactory.java index 2e8e0bae68..6f9587ac91 100644 --- a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBFactory.java +++ b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBFactory.java @@ -41,6 +41,9 @@ import org.rocksdb.MemoryUsageType; import org.rocksdb.MemoryUtil; import org.rocksdb.RocksDB; +import org.rocksdb.Status; +import org.rocksdb.TableFileCreationInfo; +import org.rocksdb.TableFileDeletionInfo; import lombok.extern.slf4j.Slf4j; @@ -49,6 +52,8 @@ public final class RocksDBFactory { private static final List rocksdbChangedListeners = new ArrayList<>(); private static RocksDBFactory dbFactory; + /** Singleton event listener wired into every new RocksDB instance. */ + private final RocksdbEventListener rocksdbEventListener = new RocksdbEventListener(); static { RocksDB.loadLibrary(); @@ -178,6 +183,46 @@ public void onCompactionCompleted(RocksDB db, CompactionJobInfo compactionJobInf public void onCompactionBegin(final RocksDB db, final CompactionJobInfo compactionJobInfo) { log.info("RocksdbEventListener onCompactionBegin"); } + + /** + * Invoked by RocksDB after a new SST (table) file has been successfully created. + * Propagates the event to all registered {@link RocksdbChangedListener}s so that + * cloud-storage providers can upload the new file. + */ + @Override + public void onTableFileCreated(final TableFileCreationInfo info) { + if (info.getStatus().getCode() != Status.Code.Ok) { + log.warn("onTableFileCreated: skipping failed file creation, path={}, status={}", + info.getFilePath(), info.getStatus().getCodeString()); + return; + } + log.debug("onTableFileCreated: db={}, cf={}, path={}, size={}", + info.getDbName(), info.getColumnFamilyName(), + info.getFilePath(), info.getFileSize()); + rocksdbChangedListeners.forEach(listener -> + listener.onTableFileCreated(info.getDbName(), info.getColumnFamilyName(), + info.getFilePath(), info.getFileSize()) + ); + } + + /** + * Invoked by RocksDB after an SST (table) file has been deleted. + * Propagates the event to all registered {@link RocksdbChangedListener}s so that + * cloud-storage providers can remove the corresponding remote object. + */ + @Override + public void onTableFileDeleted(final TableFileDeletionInfo info) { + if (info.getStatus().getCode() != Status.Code.Ok) { + log.warn("onTableFileDeleted: skipping failed file deletion, path={}, status={}", + info.getFilePath(), info.getStatus().getCodeString()); + return; + } + log.debug("onTableFileDeleted: db={}, path={}", + info.getDbName(), info.getFilePath()); + rocksdbChangedListeners.forEach(listener -> + listener.onTableFileDeleted(info.getDbName(), null, info.getFilePath()) + ); + } } public RocksDBSession createGraphDB(String dbPath, String dbName) { @@ -189,16 +234,30 @@ public RocksDBSession createGraphDB(String dbPath, String dbName, long version) throw new RuntimeException("db closed"); } operateLock.writeLock().lock(); + boolean isNew = false; + RocksDBSession dbSession = null; try { - RocksDBSession dbSession = dbSessionMap.get(dbName); + dbSession = dbSessionMap.get(dbName); if (dbSession == null) { + String dbOpenPath = dbPath.endsWith(File.separator) ? dbPath + dbName : + dbPath + File.separator + dbName; + rocksdbChangedListeners.forEach(listener -> listener.onDBOpening(dbName, dbOpenPath)); log.info("create rocksdb for {}", dbName); dbSession = new RocksDBSession(this.hugeConfig, dbPath, dbName, version); dbSessionMap.put(dbName, dbSession); + isNew = true; } return dbSession.clone(); } finally { operateLock.writeLock().unlock(); + if (isNew && dbSession != null) { + // Notify listeners so they can upload any pre-existing SST files + // and flush the MemTable (which may hold WAL-recovered data). + final String finalDbName = dbSession.getGraphName(); + final String finalDbPath = dbSession.getDbPath(); + rocksdbChangedListeners.forEach(listener -> + listener.onDBCreated(finalDbName, finalDbPath)); + } } } @@ -314,11 +373,74 @@ public void addRocksdbChangedListener(RocksdbChangedListener listener) { rocksdbChangedListeners.add(listener); } + /** + * Flushes the MemTable of the named RocksDB session to disk, creating an SST file. + * This triggers {@link RocksdbChangedListener#onTableFileCreated} for every registered + * listener (including cloud-storage upload). + * + * @param dbName the graph / partition name + * @param wait if {@code true} the call blocks until the flush completes + */ + public void flushSession(String dbName, boolean wait) { + RocksDBSession session = dbSessionMap.get(dbName); + if (session != null) { + try { + session.flush(wait); + log.debug("Flushed RocksDB session for db={}", dbName); + } catch (Exception e) { + log.warn("Failed to flush RocksDB session for db={}: {}", dbName, e.getMessage()); + } + } + } + + public boolean onReadMiss(RocksDBSession session, String table, byte[] key) { + if (session == null) { + return false; + } + boolean hydrated = false; + for (RocksdbChangedListener listener : rocksdbChangedListeners) { + try { + if (listener.onReadMiss(session, table, key)) { + hydrated = true; + } + } catch (Exception e) { + log.warn("onReadMiss listener failed for db={}, table={}: {}", + session.getGraphName(), table, e.getMessage()); + } + } + return hydrated; + } + + /** + * Returns the singleton {@link RocksdbEventListener} that should be registered + * with every new {@link org.rocksdb.DBOptions} via + * {@link org.rocksdb.DBOptions#setListeners}. + */ + public RocksdbEventListener getEventListener() { + return rocksdbEventListener; + } + public interface RocksdbChangedListener { default void onCompacted(String dbName) { } + default void onDBOpening(String dbName, String dbPath) { + } + + /** + * Called immediately after a new RocksDB instance has been opened for the first time. + * + *

Implementations can use this callback to upload any SST files that already exist + * in {@code dbPath} (e.g. from a previous run) and to trigger a MemTable flush so that + * any WAL-recovered data is also written to SST files and forwarded to cloud storage. + * + * @param dbName logical name of the graph / partition + * @param dbPath absolute path of the RocksDB directory + */ + default void onDBCreated(String dbName, String dbPath) { + } + default void onDBDeleteBegin(String dbName, String filePath) { } @@ -327,6 +449,40 @@ default void onDBDeleted(String dbName, String filePath) { default void onDBSessionReleased(RocksDBSession dbSession) { } + + /** + * Called after a new SST file has been successfully created by RocksDB. + * + * @param dbName RocksDB instance name + * @param cfName column-family name + * @param filePath absolute path of the new SST file + * @param fileSize size of the file in bytes + */ + default void onTableFileCreated(String dbName, String cfName, + String filePath, long fileSize) { + } + + /** + * Called after an SST file has been deleted by RocksDB. + * + * @param dbName RocksDB instance name + * @param cfName column-family name + * @param filePath absolute path of the deleted SST file + */ + default void onTableFileDeleted(String dbName, String cfName, String filePath) { + } + + /** + * Called when a get() operation returns null so listeners can attempt cloud hydration. + * + * @param session RocksDB session where miss happened + * @param table target column-family/table + * @param key requested key + * @return true if listener hydrated new local data and caller should retry get() + */ + default boolean onReadMiss(RocksDBSession session, String table, byte[] key) { + return false; + } } class DBSessionWatcher { diff --git a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java index f4e7605a7f..f4a1b8d8fa 100644 --- a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java +++ b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java @@ -26,6 +26,7 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -431,6 +432,11 @@ private void openRocksDB(String dbDataPath, long version) { RocksDBSession.initOptions(hugeConfig, opts, opts, opts, opts); dbOptions = new DBOptions(opts); dbOptions.setStatistics(rocksDbStats); + // Register the factory-level event listener so that table-file events + // (onTableFileCreated / onTableFileDeleted) are forwarded to all + // RocksdbChangedListener implementations (e.g. cloud storage providers). + dbOptions.setListeners( + Collections.singletonList(RocksDBFactory.getInstance().getEventListener())); try { List columnFamilyDescriptorList = diff --git a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/SessionOperatorImpl.java b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/SessionOperatorImpl.java index b8259e5220..b1b79c211c 100644 --- a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/SessionOperatorImpl.java +++ b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/SessionOperatorImpl.java @@ -208,7 +208,14 @@ public void deleteRange(byte[] keyFrom, byte[] keyTo) throws DBStoreException { @Override public byte[] get(String table, byte[] key) throws DBStoreException { try (CFHandleLock cf = this.getLock(table)) { - return rocksdb().get(cf.get(), key); + byte[] value = rocksdb().get(cf.get(), key); + if (value != null) { + return value; + } + if (RocksDBFactory.getInstance().onReadMiss(this.session, table, key)) { + return rocksdb().get(cf.get(), key); + } + return null; } catch (RocksDBException e) { throw new DBStoreException(e); } diff --git a/hugegraph-store/pom.xml b/hugegraph-store/pom.xml index 9ff1e933e5..97e2391c14 100644 --- a/hugegraph-store/pom.xml +++ b/hugegraph-store/pom.xml @@ -21,7 +21,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 hugegraph-store - ${revision} pom @@ -41,6 +40,7 @@ hg-store-node hg-store-dist hg-store-cli + hg-store-cloud-s3 @@ -80,6 +80,11 @@ hg-store-core ${project.version} + + org.apache.hugegraph + hg-store-cloud-s3 + ${project.version} + org.apache.hugegraph hg-store-transfer @@ -302,5 +307,13 @@ + + + + cloud-s3 + + hg-store-cloud-s3 + + diff --git a/install-dist/release-docs/LICENSE b/install-dist/release-docs/LICENSE index 0014317824..322fb0e50e 100644 --- a/install-dist/release-docs/LICENSE +++ b/install-dist/release-docs/LICENSE @@ -245,6 +245,14 @@ The text of each license is also included in licenses/LICENSE-[project].txt. https://central.sonatype.com/artifact/org.hdrhistogram/HdrHistogram/2.1.12 -> CC0 1.0 https://central.sonatype.com/artifact/org.hdrhistogram/HdrHistogram/2.1.9 -> CC0 1.0 +======================================================================== +Third party MIT-0 licenses +======================================================================== +The following components are provided under the MIT-0 License. See project link for details. +The text of each license is also included in licenses/LICENSE-[project].txt. + + https://central.sonatype.com/artifact/org.reactivestreams/reactive-streams/1.0.4 -> MIT-0 + ======================================================================== Third party BSD-2-Clause licenses ======================================================================== @@ -444,28 +452,38 @@ non-Apache-2.0 license text is required for a bundled component. https://central.sonatype.com/artifact/io.netty/netty-all/4.1.42.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-all/4.1.44.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-all/4.1.61.Final -> Apache 2.0 + https://central.sonatype.com/artifact/io.netty/netty-buffer/4.1.108.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-buffer/4.1.52.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-buffer/4.1.72.Final -> Apache 2.0 + https://central.sonatype.com/artifact/io.netty/netty-codec-http/4.1.108.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-codec-http/4.1.52.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-codec-http/4.1.72.Final -> Apache 2.0 + https://central.sonatype.com/artifact/io.netty/netty-codec-http2/4.1.108.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-codec-http2/4.1.52.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-codec-http2/4.1.72.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-codec-socks/4.1.52.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-codec-socks/4.1.72.Final -> Apache 2.0 + https://central.sonatype.com/artifact/io.netty/netty-codec/4.1.108.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-codec/4.1.52.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-codec/4.1.72.Final -> Apache 2.0 + https://central.sonatype.com/artifact/io.netty/netty-common/4.1.108.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-common/4.1.52.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-common/4.1.72.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-handler-proxy/4.1.52.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-handler-proxy/4.1.72.Final -> Apache 2.0 + https://central.sonatype.com/artifact/io.netty/netty-handler/4.1.108.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-handler/4.1.52.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-handler/4.1.72.Final -> Apache 2.0 + https://central.sonatype.com/artifact/io.netty/netty-resolver/4.1.108.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-resolver/4.1.52.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-resolver/4.1.72.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-tcnative-boringssl-static/2.0.25.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-tcnative-boringssl-static/2.0.36.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-tcnative-classes/2.0.46.Final -> Apache 2.0 + https://central.sonatype.com/artifact/io.netty/netty-transport-classes-epoll/4.1.108.Final -> Apache 2.0 + https://central.sonatype.com/artifact/io.netty/netty-transport-native-unix-common/4.1.108.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-transport-native-unix-common/4.1.72.Final -> Apache 2.0 + https://central.sonatype.com/artifact/io.netty/netty-transport/4.1.108.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-transport/4.1.52.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.netty/netty-transport/4.1.72.Final -> Apache 2.0 https://central.sonatype.com/artifact/io.opentracing/opentracing-api/0.22.0 -> Apache 2.0 @@ -740,6 +758,33 @@ non-Apache-2.0 license text is required for a bundled component. https://central.sonatype.com/artifact/org.yaml/snakeyaml/1.28 -> Apache 2.0 https://central.sonatype.com/artifact/org.yaml/snakeyaml/2.2 -> Apache 2.0 https://central.sonatype.com/artifact/org.zeroturnaround/zt-zip/1.14 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/annotations/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/apache-client/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/arns/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/auth/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/aws-core/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/aws-query-protocol/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/aws-xml-protocol/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/checksums-spi/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/checksums/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/crt-core/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/endpoints-spi/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/http-auth-aws/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/http-auth-spi/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/http-auth/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/http-client-spi/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/identity-spi/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/json-utils/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/metrics-spi/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/netty-nio-client/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/profiles/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/protocol-core/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/regions/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/s3/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/sdk-core/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/third-party-jackson-core/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/utils/2.25.60 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.eventstream/eventstream/1.0.1 -> Apache 2.0 ======================================================================== Third party BSD licenses diff --git a/install-dist/release-docs/NOTICE b/install-dist/release-docs/NOTICE index 775f03241c..6cddb47b85 100644 --- a/install-dist/release-docs/NOTICE +++ b/install-dist/release-docs/NOTICE @@ -1890,6 +1890,38 @@ Copyright 2020-2021 SmartBear Software Inc. ======================================================================== +AWS SDK for Java 2.0 NOTICE + +======================================================================== + +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). + +======================================================================== + # # jansi 2.4.0 (https://github.com/fusesource/jansi) # diff --git a/install-dist/release-docs/licenses/LICENSE-reactive-streams-1.0.4.txt b/install-dist/release-docs/licenses/LICENSE-reactive-streams-1.0.4.txt new file mode 100644 index 0000000000..cfcf3bd994 --- /dev/null +++ b/install-dist/release-docs/licenses/LICENSE-reactive-streams-1.0.4.txt @@ -0,0 +1,17 @@ +MIT No Attribution + +Copyright (c) 2015-2022 Reactive Streams contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/install-dist/scripts/dependency/known-dependencies.txt b/install-dist/scripts/dependency/known-dependencies.txt index 326ac3dd76..8936446ca8 100644 --- a/install-dist/scripts/dependency/known-dependencies.txt +++ b/install-dist/scripts/dependency/known-dependencies.txt @@ -10,12 +10,15 @@ animal-sniffer-annotations-1.14.jar animal-sniffer-annotations-1.18.jar animal-sniffer-annotations-1.19.jar annotations-13.0.jar +annotations-2.33.8.jar annotations-24.0.1.jar annotations-4.1.1.4.jar ansj_seg-5.1.6.jar antlr-runtime-3.5.2.jar aopalliance-repackaged-3.0.1.jar +apache-client-2.33.8.jar apiguardian-api-1.1.0.jar +arns-2.33.8.jar arthas-agent-attach-3.6.4.jar arthas-agent-attach-3.7.1.jar arthas-packaging-3.6.4.jar @@ -33,8 +36,12 @@ asm-util-5.0.3.jar assertj-core-3.19.0.jar ast-9.0-9.0.20190305.jar audience-annotations-0.13.0.jar +auth-2.33.8.jar auto-service-annotations-1.0.jar automaton-1.11-8.jar +aws-core-2.33.8.jar +aws-query-protocol-2.33.8.jar +aws-xml-protocol-2.33.8.jar bolt-1.6.2.jar bolt-1.6.4.jar byte-buddy-1.10.20.jar @@ -51,6 +58,8 @@ checker-qual-2.0.0.jar checker-qual-3.12.0.jar checker-qual-3.33.0.jar checker-qual-3.5.0.jar +checksums-2.33.8.jar +checksums-spi-2.33.8.jar chronicle-bytes-2.20.111.jar chronicle-core-2.20.126.jar chronicle-queue-5.20.123.jar @@ -63,6 +72,7 @@ commons-cli-1.5.0.jar commons-codec-1.11.jar commons-codec-1.13.jar commons-codec-1.15.jar +commons-codec-1.17.1.jar commons-codec-1.9.jar commons-collections-3.2.2.jar commons-collections4-4.4.jar @@ -85,6 +95,7 @@ commons-pool2-2.0.jar commons-text-1.10.0.jar commons-text-1.9.jar concurrent-trees-2.4.0.jar +crt-core-2.33.8.jar cypher-gremlin-extensions-1.0.4.jar disruptor-3.3.7.jar disruptor-3.4.1.jar @@ -92,12 +103,14 @@ eclipse-collections-10.4.0.jar eclipse-collections-11.1.0.jar eclipse-collections-api-10.4.0.jar eclipse-collections-api-11.1.0.jar +endpoints-spi-2.33.8.jar error_prone_annotations-2.1.3.jar error_prone_annotations-2.10.0.jar error_prone_annotations-2.18.0.jar error_prone_annotations-2.3.4.jar error_prone_annotations-2.4.0.jar error_prone_annotations-2.48.0.jar +eventstream-1.0.1.jar exp4j-0.4.8.jar expressions-9.0-9.0.20190305.jar failsafe-2.4.1.jar @@ -193,8 +206,15 @@ hk2-utils-3.0.1.jar hppc-0.7.1.jar hppc-0.8.1.jar htrace-core4-4.1.0-incubating.jar +http-auth-2.33.8.jar +http-auth-aws-2.33.8.jar +http-auth-aws-eventstream-2.33.8.jar +http-auth-spi-2.33.8.jar +http-client-spi-2.33.8.jar httpclient-4.5.13.jar httpcore-4.4.13.jar +httpcore-4.4.16.jar +identity-spi-2.33.8.jar ikanalyzer-2012_u6.jar ivy-2.4.0.jar j2objc-annotations-1.1.jar @@ -324,6 +344,7 @@ jraft-core-1.3.9.jar json-path-2.5.0.jar json-simple-1.1.jar json-smart-2.3.jar +json-utils-2.33.8.jar jsonassert-1.5.0.jar jsr305-3.0.1.jar jsr305-3.0.2.jar @@ -421,6 +442,7 @@ metrics-core-4.2.4.jar metrics-jersey3-4.2.4.jar metrics-jvm-3.1.5.jar metrics-logback-3.1.5.jar +metrics-spi-2.33.8.jar micrometer-core-1.7.12.jar micrometer-registry-prometheus-1.7.12.jar mmseg4j-core-1.10.0.jar @@ -431,33 +453,44 @@ mxdump-0.14.jar netty-all-4.1.42.Final.jar netty-all-4.1.44.Final.jar netty-all-4.1.61.Final.jar +netty-buffer-4.1.126.Final.jar netty-buffer-4.1.52.Final.jar netty-buffer-4.1.72.Final.jar +netty-codec-4.1.126.Final.jar netty-codec-4.1.52.Final.jar netty-codec-4.1.72.Final.jar +netty-codec-http-4.1.126.Final.jar netty-codec-http-4.1.52.Final.jar netty-codec-http-4.1.72.Final.jar +netty-codec-http2-4.1.126.Final.jar netty-codec-http2-4.1.52.Final.jar netty-codec-http2-4.1.72.Final.jar netty-codec-socks-4.1.52.Final.jar netty-codec-socks-4.1.72.Final.jar +netty-common-4.1.126.Final.jar netty-common-4.1.52.Final.jar netty-common-4.1.72.Final.jar +netty-handler-4.1.126.Final.jar netty-handler-4.1.130.Final.jar netty-handler-4.1.52.Final.jar netty-handler-4.1.72.Final.jar netty-handler-proxy-4.1.52.Final.jar netty-handler-proxy-4.1.72.Final.jar +netty-nio-client-2.33.8.jar +netty-resolver-4.1.126.Final.jar netty-resolver-4.1.130.Final.jar netty-resolver-4.1.52.Final.jar netty-resolver-4.1.72.Final.jar netty-tcnative-boringssl-static-2.0.25.Final.jar netty-tcnative-boringssl-static-2.0.36.Final.jar netty-tcnative-classes-2.0.46.Final.jar +netty-transport-4.1.126.Final.jar netty-transport-4.1.52.Final.jar netty-transport-4.1.72.Final.jar +netty-transport-classes-epoll-4.1.126.Final.jar netty-transport-classes-epoll-4.1.130.Final.jar netty-transport-native-epoll-4.1.130.Final.jar +netty-transport-native-unix-common-4.1.126.Final.jar netty-transport-native-unix-common-4.1.72.Final.jar nimbus-jose-jwt-4.41.2.jar nlp-lang-1.7.7.jar @@ -496,6 +529,7 @@ powermock-module-junit4-2.0.0-RC.3.jar powermock-module-junit4-common-2.0.0-RC.3.jar powermock-module-junit4-rule-2.0.0-RC.3.jar powermock-reflect-2.0.0-RC.3.jar +profiles-2.33.8.jar proto-google-common-protos-1.17.0.jar proto-google-common-protos-2.0.1.jar protobuf-java-3.11.0.jar @@ -503,20 +537,27 @@ protobuf-java-3.17.2.jar protobuf-java-3.21.7.jar protobuf-java-3.5.1.jar protobuf-java-util-3.17.2.jar +protocol-core-2.33.8.jar protostuff-api-1.6.0.jar protostuff-collectionschema-1.6.0.jar protostuff-core-1.6.0.jar protostuff-runtime-1.6.0.jar psjava-0.1.19.jar +reactive-streams-1.0.4.jar +regions-2.33.8.jar reporter-config-base-3.0.3.jar reporter-config3-3.0.3.jar +retries-2.33.8.jar +retries-spi-2.33.8.jar rewriting-9.0-9.0.20190305.jar rocksdbjni-6.29.5.jar rocksdbjni-7.7.3.jar rocksdbjni-8.10.2.jar +s3-2.33.8.jar scala-java8-compat_2.12-0.8.0.jar scala-library-2.12.7.jar scala-reflect-2.12.7.jar +sdk-core-2.33.8.jar shims-0.9.38.jar sigar-1.6.4.jar simpleclient-0.10.0.jar @@ -539,6 +580,7 @@ slf4j-api-1.7.21.jar slf4j-api-1.7.25.jar slf4j-api-1.7.31.jar slf4j-api-1.7.32.jar +slf4j-api-1.7.36.jar slf4j-api-2.0.9.jar snakeyaml-1.18.jar snakeyaml-1.26.jar @@ -591,12 +633,15 @@ swagger-integration-jakarta-2.2.18.jar swagger-jaxrs2-jakarta-2.2.18.jar swagger-models-1.5.18.jar swagger-models-jakarta-2.2.18.jar +third-party-jackson-core-2.33.8.jar tinkergraph-gremlin-3.5.1.jar token-provider-2.0.0.jar tomcat-embed-el-9.0.63.jar tracer-core-3.0.8.jar translation-1.0.4.jar +url-connection-client-2.33.8.jar util-9.0-9.0.20190305.jar +utils-2.33.8.jar validation-api-1.1.0.Final.jar websocket-api-9.4.46.v20220331.jar websocket-client-9.4.46.v20220331.jar diff --git a/install-dist/scripts/dependency/regenerate_known_dependencies.sh b/install-dist/scripts/dependency/regenerate_known_dependencies.sh old mode 100644 new mode 100755 diff --git a/pom.xml b/pom.xml index 850ac99fa8..0b5044ba81 100644 --- a/pom.xml +++ b/pom.xml @@ -217,6 +217,10 @@ **/pid **/tmp/** + + docker/cloud-storage/** + + **/.generated/** **/src/main/java/org/apache/hugegraph/pd/grpc/** **/src/main/java/org/apache/hugegraph/store/grpc/** @@ -311,6 +315,9 @@ **/*.txt **/.flattened-pom.xml **/apache-hugegraph-*/**/* + + docker/cloud-storage/** + **/.generated/**