Data fusion changes

.github/workflows/datafusion-e2e-test.yml

@@ -0,0 +1,205 @@
+name: DataFusion E2E Integration Test
+
+on:
+  pull_request:
+    branches:
+      - feature/substrait-plan
+  push:
+    branches:
+      - feature/substrait-plan
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    env:
+      RUSTFLAGS: "-A unused_variables -A unused_mut"
+
+    steps:
+      - name: Checkout OpenSearch
+        uses: actions/checkout@v4
+        with:
+          repository: opensearch-project/OpenSearch
+          ref: feature/datafusion
+          path: OpenSearch
+
+      - name: Set up JDK
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '21'
+
+      - name: Setup Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          components: rustfmt
+
+      - name: Install Protocol Buffers
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y protobuf-compiler
+
+      - name: Install Protocol Buffers
+        if: runner.os == 'macOS'
+        run: brew install protobuf
+
+      - name: Install Protocol Buffers
+        if: runner.os == 'Windows'
+        run: choco install protoc
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v3
+
+      - name: Publish OpenSearch to Maven Local
+        working-directory: OpenSearch
+        run: ./gradlew publishToMavenLocal -x test -PrustDebug=true
+
+      - name: Checkout SQL Plugin
+        uses: actions/checkout@v4
+        with:
+          path: opensearch-sql
+
+      - name: Publish SQL Plugin to Maven Local
+        working-directory: opensearch-sql
+        run: ./gradlew publishToMavenLocal -x test
+
+      - name: Run DataFusionReaderManager Tests
+        working-directory: OpenSearch
+        run: ./gradlew :plugins:engine-datafusion:test --tests "org.opensearch.datafusion.DataFusionReaderManagerTests"
+
+      - name: Run OpenSearch with DataFusion Plugin
+        working-directory: OpenSearch
+        run: |
+          ./gradlew run \
+            --preserve-data \
+            -PremotePlugins="['org.opensearch.plugin:opensearch-job-scheduler:3.3.0.0-SNAPSHOT', 'org.opensearch.plugin:opensearch-sql-plugin:3.3.0.0-SNAPSHOT']" \
+            -PinstalledPlugins="['engine-datafusion']" &
+
+          # Wait for OpenSearch to start
+          timeout 300 bash -c 'until curl -s http://localhost:9200; do sleep 5; done'
+
+      - name: Run SQL CalcitePPLClickbenchIT
+        working-directory: opensearch-sql
+        run: ./gradlew :integ-test:integTest --tests "org.opensearch.sql.calcite.clickbench.CalcitePPLClickBenchIT" -Dtests.method="testDataFusion" -Dtests.cluster=localhost:9200 -Dtests.rest.cluster=localhost:9200 -DignorePrometheus=true -Dtests.clustername=opensearch -Dtests.output=true
+
+      - name: Run E2E Tests
+        run: |
+          # Create index
+          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X PUT 'http://localhost:9200/index-7' \
+            -H 'Content-Type: application/json' \
+            -d '{
+              "settings": {
+                "number_of_shards": 1,
+                "number_of_replicas": 0,
+                "refresh_interval": -1
+              },
+              "mappings": {
+                "properties": {
+                  "id": {"type": "keyword"},
+                  "name": {"type": "keyword"},
+                  "age": {"type": "integer"},
+                  "salary": {"type": "long"},
+                  "score": {"type": "double"},
+                  "active": {"type": "boolean"},
+                  "created_date": {"type": "date"}
+                }
+              }
+            }')
+          if [ "$HTTP_CODE" != "200" ]; then
+            echo "Failed to create index. HTTP status: $HTTP_CODE"
+            exit 1
+          fi
+
+          # Index documents
+          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST 'http://localhost:9200/_bulk' \
+            -H 'Content-Type: application/json' \
+            -d '{"index":{"_index":"index-7"}}
+                {"id":"1","name":"Alice","age":30,"salary":75000,"score":95.5,"active":true,"created_date":"2024-01-15"}
+                {"index":{"_index":"index-7"}}
+                {"id":"2","name":"Bob","age":25,"salary":60000,"score":88.3,"active":true,"created_date":"2024-02-20"}
+                {"index":{"_index":"index-7"}}
+                {"id":"3","name":"Charlie","age":35,"salary":90000,"score":92.7,"active":false,"created_date":"2024-03-10"}
+                {"index":{"_index":"index-7"}}
+                {"id":"4","name":"Diana","age":28,"salary":70000,"score":89.1,"active":true,"created_date":"2024-04-05"}
+                {"index":{"_index":"index-7"}}
+                {"id":"5","name":"Bob","age":30,"salary":55000,"score":81.1,"active":true,"created_date":"2024-04-05"}
+                {"index":{"_index":"index-7"}}
+                {"id":"6","name":"Diana","age":35,"salary":65000,"score":71.1,"active":true,"created_date":"2024-02-05"}
+          ')
+          if [ "$HTTP_CODE" != "200" ]; then
+            echo "Failed to index documents. HTTP status: $HTTP_CODE"
+            exit 1
+          fi
+
+          # Refresh index
+          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST 'localhost:9200/index-7/_refresh')
+          if [ "$HTTP_CODE" != "200" ]; then
+            echo "Failed to refresh index. HTTP status: $HTTP_CODE"
+            exit 1
+          fi
+
+          # Query 1: stats with min, max, avg
+          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST 'http://localhost:9200/_plugins/_ppl' \
+            -H 'Content-Type: application/json' \
+            -d '{"query": "source=index-7 | stats count(), min(age) as min, max(age) as max, avg(age) as avg"}')
+          if [ "$HTTP_CODE" != "200" ]; then
+            echo "Query 1 failed. HTTP status: $HTTP_CODE"
+            exit 1
+          fi
+
+          # Query 2: count by name with sort
+          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST 'http://localhost:9200/_plugins/_ppl' \
+            -H 'Content-Type: application/json' \
+            -d '{"query": "source=index-7 | stats count() as c by name | sort c"}')
+          if [ "$HTTP_CODE" != "200" ]; then
+            echo "Query 2 failed. HTTP status: $HTTP_CODE"
+            exit 1
+          fi
+
+          # Query 3: count and sum by name with sort
+          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST 'http://localhost:9200/_plugins/_ppl' \
+            -H 'Content-Type: application/json' \
+            -d '{"query": "source=index-7 | stats count(), sum(age) as c by name | sort c"}')
+          if [ "$HTTP_CODE" != "200" ]; then
+            echo "Query 3 failed. HTTP status: $HTTP_CODE"
+            exit 1
+          fi
+
+          # Query 4: where clause with stats
+          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST 'http://localhost:9200/_plugins/_ppl' \
+            -H 'Content-Type: application/json' \
+            -d '{"query": "source=index-7 | where name = \"Bob\" | stats sum(age)"}')
+          if [ "$HTTP_CODE" != "200" ]; then
+            echo "Query 4 failed. HTTP status: $HTTP_CODE"
+            exit 1
+          fi
+
+          # Query 5: sum by name sorted by sum
+          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST 'http://localhost:9200/_plugins/_ppl' \
+            -H 'Content-Type: application/json' \
+            -d '{"query": "source=index-7 | stats sum(age) as s by name | sort s"}')
+          if [ "$HTTP_CODE" != "200" ]; then
+            echo "Query 5 failed. HTTP status: $HTTP_CODE"
+            exit 1
+          fi
+
+          # Query 6: sum by name sorted by name
+          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST 'http://localhost:9200/_plugins/_ppl' \
+            -H 'Content-Type: application/json' \
+            -d '{"query": "source=index-7 | stats sum(age) as s by name | sort name"}')
+          if [ "$HTTP_CODE" != "200" ]; then
+            echo "Query 6 failed. HTTP status: $HTTP_CODE"
+            exit 1
+          fi
+
+          # Query 7: count by name
+          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST 'http://localhost:9200/_plugins/_ppl' \
+            -H 'Content-Type: application/json' \
+            -d '{"query": "source=index-7 | stats count() as c by name"}')
+          if [ "$HTTP_CODE" != "200" ]; then
+            echo "Query 7 failed. HTTP status: $HTTP_CODE"
+            exit 1
+          fi
+
+          echo "All E2E tests passed successfully!"

.github/workflows/delete_backport_branch.yml

@@ -7,9 +7,16 @@ on:
 jobs:
   delete-branch:
     runs-on: ubuntu-latest
-    if: startsWith(github.event.pull_request.head.ref,'backport/')
+    permissions:
+      pull-requests: write
+    if: startsWith(github.event.pull_request.head.ref,'backport/') || startsWith(github.event.pull_request.head.ref,'release-chores/')
     steps:
       - name: Delete merged branch
-        uses: SvanBoxel/delete-merged-branch@main
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        uses: actions/github-script@v7
+        with:
+          script: |
+            github.rest.git.deleteRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: `heads/${context.payload.pull_request.head.ref}`,
+            })

.github/workflows/maven-publish-modules.yml

@@ -0,0 +1,47 @@
+name: Publish unified query modules to maven
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.x'
+
+env:
+  SNAPSHOT_REPO_URL: https://ci.opensearch.org/ci/dbc/snapshots/maven/
+
+jobs:
+  publish-unified-query-modules:
+    strategy:
+      fail-fast: false
+    if: github.repository == 'opensearch-project/sql'
+    runs-on: ubuntu-latest
+
+    permissions:
+      id-token: write
+      contents: write
+
+    steps:
+      - uses: actions/setup-java@v3
+        with:
+          distribution: temurin # Temurin is a distribution of adoptium
+          java-version: 21
+      - uses: actions/checkout@v3
+      - name: Load secret
+        uses: 1password/load-secrets-action@v2
+        with:
+          # Export loaded secrets as environment variables
+          export-env: true
+        env:
+          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+          MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
+          MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
+          aws-region: us-east-1
+      - name: publish snapshots to maven
+        run: |
+          ./gradlew publishUnifiedQueryPublicationToSnapshotsRepository

.github/workflows/maven-publish.yml

@@ -8,9 +8,6 @@ on:
       - '[0-9]+.[0-9]+'
       - '[0-9]+.x'
 
-env:
-  SNAPSHOT_REPO_URL: https://ci.opensearch.org/ci/dbc/snapshots/maven/
-
 jobs:
   build-and-publish-snapshots:
     strategy:

.github/workflows/publish-async-query-core.yml

@@ -5,8 +5,8 @@ on:
   push:
     branches:
       - main
-      - 1.*
-      - 2.*
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.x'
     paths:
       - 'async-query-core/**'
       - '.github/workflows/publish-async-query-core.yml'
@@ -18,7 +18,6 @@ concurrency:
   cancel-in-progress: false
 
 env:
-  SNAPSHOT_REPO_URL: https://central.sonatype.com/repository/maven-snapshots/
   COMMIT_MAP_FILENAME: commit-history-async-query-core.json
 
 jobs:
@@ -47,8 +46,19 @@ jobs:
           export-env: true
         env:
           OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
-          SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username
-          SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password
+          MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
+          MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
+
+      - name: Export SNAPSHOT_REPO_URL
+        run: |
+          snapshot_repo_url=${{ env.MAVEN_SNAPSHOTS_S3_REPO }}
+          echo "SNAPSHOT_REPO_URL=$snapshot_repo_url" >> $GITHUB_ENV
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
+          aws-region: us-east-1
 
       - name: Set commit ID
         id: set_commit
@@ -83,4 +93,4 @@ jobs:
           source ./.github/maven-publish-utils.sh
 
           # Call the main function for async-query-core
-          publish_async_query_core "${{ steps.extract_version.outputs.VERSION }}" "${{ steps.set_commit.outputs.commit_id }}"
+          publish_async_query_core "${{ steps.extract_version.outputs.VERSION }}" "${{ steps.set_commit.outputs.commit_id }}"

.github/workflows/publish-grammar-files.yml

@@ -5,8 +5,8 @@ on:
   push:
     branches:
       - main
-      - 1.*
-      - 2.*
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.x'
     paths:
       - 'language-grammar/src/main/antlr4/**'
       - 'language-grammar/build.gradle'
@@ -19,7 +19,6 @@ concurrency:
   cancel-in-progress: false
 
 env:
-  SNAPSHOT_REPO_URL: https://central.sonatype.com/repository/maven-snapshots/
   COMMIT_MAP_FILENAME: commit-history-language-grammar.json
 
 jobs:
@@ -51,8 +50,19 @@ jobs:
           export-env: true
         env:
           OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
-          SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username
-          SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password
+          MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
+          MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
+
+      - name: Export SNAPSHOT_REPO_URL
+        run: |
+          snapshot_repo_url=${{ env.MAVEN_SNAPSHOTS_S3_REPO }}
+          echo "SNAPSHOT_REPO_URL=$snapshot_repo_url" >> $GITHUB_ENV
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
+          aws-region: us-east-1
 
       - name: Set version
         id: set_version