diff --git a/.github/workflows/datafusion-e2e-test.yml b/.github/workflows/datafusion-e2e-test.yml new file mode 100644 index 00000000000..3c6e8fc3170 --- /dev/null +++ b/.github/workflows/datafusion-e2e-test.yml @@ -0,0 +1,205 @@ +name: DataFusion E2E Integration Test + +on: + pull_request: + branches: + - feature/substrait-plan + push: + branches: + - feature/substrait-plan + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-latest + env: + RUSTFLAGS: "-A unused_variables -A unused_mut" + + steps: + - name: Checkout OpenSearch + uses: actions/checkout@v4 + with: + repository: opensearch-project/OpenSearch + ref: feature/datafusion + path: OpenSearch + + - name: Set up JDK + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '21' + + - name: Setup Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + components: rustfmt + + - name: Install Protocol Buffers + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler + + - name: Install Protocol Buffers + if: runner.os == 'macOS' + run: brew install protobuf + + - name: Install Protocol Buffers + if: runner.os == 'Windows' + run: choco install protoc + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v3 + + - name: Publish OpenSearch to Maven Local + working-directory: OpenSearch + run: ./gradlew publishToMavenLocal -x test -PrustDebug=true + + - name: Checkout SQL Plugin + uses: actions/checkout@v4 + with: + path: opensearch-sql + + - name: Publish SQL Plugin to Maven Local + working-directory: opensearch-sql + run: ./gradlew publishToMavenLocal -x test + + - name: Run DataFusionReaderManager Tests + working-directory: OpenSearch + run: ./gradlew :plugins:engine-datafusion:test --tests "org.opensearch.datafusion.DataFusionReaderManagerTests" + + - name: Run OpenSearch with DataFusion Plugin + working-directory: OpenSearch + run: | + ./gradlew run \ + --preserve-data \ + -PremotePlugins="['org.opensearch.plugin:opensearch-job-scheduler:3.3.0.0-SNAPSHOT', 'org.opensearch.plugin:opensearch-sql-plugin:3.3.0.0-SNAPSHOT']" \ + -PinstalledPlugins="['engine-datafusion']" & + + # Wait for OpenSearch to start + timeout 300 bash -c 'until curl -s http://localhost:9200; do sleep 5; done' + + - name: Run SQL CalcitePPLClickbenchIT + working-directory: opensearch-sql + run: ./gradlew :integ-test:integTest --tests "org.opensearch.sql.calcite.clickbench.CalcitePPLClickBenchIT" -Dtests.method="testDataFusion" -Dtests.cluster=localhost:9200 -Dtests.rest.cluster=localhost:9200 -DignorePrometheus=true -Dtests.clustername=opensearch -Dtests.output=true + + - name: Run E2E Tests + run: | + # Create index + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X PUT 'http://localhost:9200/index-7' \ + -H 'Content-Type: application/json' \ + -d '{ + "settings": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": -1 + }, + "mappings": { + "properties": { + "id": {"type": "keyword"}, + "name": {"type": "keyword"}, + "age": {"type": "integer"}, + "salary": {"type": "long"}, + "score": {"type": "double"}, + "active": {"type": "boolean"}, + "created_date": {"type": "date"} + } + } + }') + if [ "$HTTP_CODE" != "200" ]; then + echo "Failed to create index. HTTP status: $HTTP_CODE" + exit 1 + fi + + # Index documents + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST 'http://localhost:9200/_bulk' \ + -H 'Content-Type: application/json' \ + -d '{"index":{"_index":"index-7"}} + {"id":"1","name":"Alice","age":30,"salary":75000,"score":95.5,"active":true,"created_date":"2024-01-15"} + {"index":{"_index":"index-7"}} + {"id":"2","name":"Bob","age":25,"salary":60000,"score":88.3,"active":true,"created_date":"2024-02-20"} + {"index":{"_index":"index-7"}} + {"id":"3","name":"Charlie","age":35,"salary":90000,"score":92.7,"active":false,"created_date":"2024-03-10"} + {"index":{"_index":"index-7"}} + {"id":"4","name":"Diana","age":28,"salary":70000,"score":89.1,"active":true,"created_date":"2024-04-05"} + {"index":{"_index":"index-7"}} + {"id":"5","name":"Bob","age":30,"salary":55000,"score":81.1,"active":true,"created_date":"2024-04-05"} + {"index":{"_index":"index-7"}} + {"id":"6","name":"Diana","age":35,"salary":65000,"score":71.1,"active":true,"created_date":"2024-02-05"} + ') + if [ "$HTTP_CODE" != "200" ]; then + echo "Failed to index documents. HTTP status: $HTTP_CODE" + exit 1 + fi + + # Refresh index + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST 'localhost:9200/index-7/_refresh') + if [ "$HTTP_CODE" != "200" ]; then + echo "Failed to refresh index. HTTP status: $HTTP_CODE" + exit 1 + fi + + # Query 1: stats with min, max, avg + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST 'http://localhost:9200/_plugins/_ppl' \ + -H 'Content-Type: application/json' \ + -d '{"query": "source=index-7 | stats count(), min(age) as min, max(age) as max, avg(age) as avg"}') + if [ "$HTTP_CODE" != "200" ]; then + echo "Query 1 failed. HTTP status: $HTTP_CODE" + exit 1 + fi + + # Query 2: count by name with sort + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST 'http://localhost:9200/_plugins/_ppl' \ + -H 'Content-Type: application/json' \ + -d '{"query": "source=index-7 | stats count() as c by name | sort c"}') + if [ "$HTTP_CODE" != "200" ]; then + echo "Query 2 failed. HTTP status: $HTTP_CODE" + exit 1 + fi + + # Query 3: count and sum by name with sort + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST 'http://localhost:9200/_plugins/_ppl' \ + -H 'Content-Type: application/json' \ + -d '{"query": "source=index-7 | stats count(), sum(age) as c by name | sort c"}') + if [ "$HTTP_CODE" != "200" ]; then + echo "Query 3 failed. HTTP status: $HTTP_CODE" + exit 1 + fi + + # Query 4: where clause with stats + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST 'http://localhost:9200/_plugins/_ppl' \ + -H 'Content-Type: application/json' \ + -d '{"query": "source=index-7 | where name = \"Bob\" | stats sum(age)"}') + if [ "$HTTP_CODE" != "200" ]; then + echo "Query 4 failed. HTTP status: $HTTP_CODE" + exit 1 + fi + + # Query 5: sum by name sorted by sum + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST 'http://localhost:9200/_plugins/_ppl' \ + -H 'Content-Type: application/json' \ + -d '{"query": "source=index-7 | stats sum(age) as s by name | sort s"}') + if [ "$HTTP_CODE" != "200" ]; then + echo "Query 5 failed. HTTP status: $HTTP_CODE" + exit 1 + fi + + # Query 6: sum by name sorted by name + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST 'http://localhost:9200/_plugins/_ppl' \ + -H 'Content-Type: application/json' \ + -d '{"query": "source=index-7 | stats sum(age) as s by name | sort name"}') + if [ "$HTTP_CODE" != "200" ]; then + echo "Query 6 failed. HTTP status: $HTTP_CODE" + exit 1 + fi + + # Query 7: count by name + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST 'http://localhost:9200/_plugins/_ppl' \ + -H 'Content-Type: application/json' \ + -d '{"query": "source=index-7 | stats count() as c by name"}') + if [ "$HTTP_CODE" != "200" ]; then + echo "Query 7 failed. HTTP status: $HTTP_CODE" + exit 1 + fi + + echo "All E2E tests passed successfully!" diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml index 387a124b8cb..67c91f68b26 100644 --- a/.github/workflows/delete_backport_branch.yml +++ b/.github/workflows/delete_backport_branch.yml @@ -7,9 +7,16 @@ on: jobs: delete-branch: runs-on: ubuntu-latest - if: startsWith(github.event.pull_request.head.ref,'backport/') + permissions: + pull-requests: write + if: startsWith(github.event.pull_request.head.ref,'backport/') || startsWith(github.event.pull_request.head.ref,'release-chores/') steps: - name: Delete merged branch - uses: SvanBoxel/delete-merged-branch@main - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: actions/github-script@v7 + with: + script: | + github.rest.git.deleteRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: `heads/${context.payload.pull_request.head.ref}`, + }) diff --git a/.github/workflows/maven-publish-modules.yml b/.github/workflows/maven-publish-modules.yml new file mode 100644 index 00000000000..64743356e97 --- /dev/null +++ b/.github/workflows/maven-publish-modules.yml @@ -0,0 +1,47 @@ +name: Publish unified query modules to maven + +on: + workflow_dispatch: + push: + branches: + - main + - '[0-9]+.[0-9]+' + - '[0-9]+.x' + +env: + SNAPSHOT_REPO_URL: https://ci.opensearch.org/ci/dbc/snapshots/maven/ + +jobs: + publish-unified-query-modules: + strategy: + fail-fast: false + if: github.repository == 'opensearch-project/sql' + runs-on: ubuntu-latest + + permissions: + id-token: write + contents: write + + steps: + - uses: actions/setup-java@v3 + with: + distribution: temurin # Temurin is a distribution of adoptium + java-version: 21 + - uses: actions/checkout@v3 + - name: Load secret + uses: 1password/load-secrets-action@v2 + with: + # Export loaded secrets as environment variables + export-env: true + env: + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo + MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v5 + with: + role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }} + aws-region: us-east-1 + - name: publish snapshots to maven + run: | + ./gradlew publishUnifiedQueryPublicationToSnapshotsRepository diff --git a/.github/workflows/maven-publish.yml b/.github/workflows/maven-publish.yml index 148baa073cb..5523647848d 100644 --- a/.github/workflows/maven-publish.yml +++ b/.github/workflows/maven-publish.yml @@ -8,9 +8,6 @@ on: - '[0-9]+.[0-9]+' - '[0-9]+.x' -env: - SNAPSHOT_REPO_URL: https://ci.opensearch.org/ci/dbc/snapshots/maven/ - jobs: build-and-publish-snapshots: strategy: diff --git a/.github/workflows/publish-async-query-core.yml b/.github/workflows/publish-async-query-core.yml index 8433cc873a3..d251851ed12 100644 --- a/.github/workflows/publish-async-query-core.yml +++ b/.github/workflows/publish-async-query-core.yml @@ -5,8 +5,8 @@ on: push: branches: - main - - 1.* - - 2.* + - '[0-9]+.[0-9]+' + - '[0-9]+.x' paths: - 'async-query-core/**' - '.github/workflows/publish-async-query-core.yml' @@ -18,7 +18,6 @@ concurrency: cancel-in-progress: false env: - SNAPSHOT_REPO_URL: https://central.sonatype.com/repository/maven-snapshots/ COMMIT_MAP_FILENAME: commit-history-async-query-core.json jobs: @@ -47,8 +46,19 @@ jobs: export-env: true env: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username - SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password + MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo + MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role + + - name: Export SNAPSHOT_REPO_URL + run: | + snapshot_repo_url=${{ env.MAVEN_SNAPSHOTS_S3_REPO }} + echo "SNAPSHOT_REPO_URL=$snapshot_repo_url" >> $GITHUB_ENV + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v5 + with: + role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }} + aws-region: us-east-1 - name: Set commit ID id: set_commit @@ -83,4 +93,4 @@ jobs: source ./.github/maven-publish-utils.sh # Call the main function for async-query-core - publish_async_query_core "${{ steps.extract_version.outputs.VERSION }}" "${{ steps.set_commit.outputs.commit_id }}" \ No newline at end of file + publish_async_query_core "${{ steps.extract_version.outputs.VERSION }}" "${{ steps.set_commit.outputs.commit_id }}" diff --git a/.github/workflows/publish-grammar-files.yml b/.github/workflows/publish-grammar-files.yml index 47ddefa1a04..7a691f250a4 100644 --- a/.github/workflows/publish-grammar-files.yml +++ b/.github/workflows/publish-grammar-files.yml @@ -5,8 +5,8 @@ on: push: branches: - main - - 1.* - - 2.* + - '[0-9]+.[0-9]+' + - '[0-9]+.x' paths: - 'language-grammar/src/main/antlr4/**' - 'language-grammar/build.gradle' @@ -19,7 +19,6 @@ concurrency: cancel-in-progress: false env: - SNAPSHOT_REPO_URL: https://central.sonatype.com/repository/maven-snapshots/ COMMIT_MAP_FILENAME: commit-history-language-grammar.json jobs: @@ -51,8 +50,19 @@ jobs: export-env: true env: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username - SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password + MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo + MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role + + - name: Export SNAPSHOT_REPO_URL + run: | + snapshot_repo_url=${{ env.MAVEN_SNAPSHOTS_S3_REPO }} + echo "SNAPSHOT_REPO_URL=$snapshot_repo_url" >> $GITHUB_ENV + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v5 + with: + role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }} + aws-region: us-east-1 - name: Set version id: set_version diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index ba1ff29b3d7..a671420fd8f 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -179,7 +179,7 @@ jobs: doctest/build/testclusters/docTestCluster-0/logs/* integ-test/build/testclusters/*/logs/* - bwc-tests: + bwc-tests-rolling-upgrade: needs: Get-CI-Image-Tag runs-on: ubuntu-latest strategy: @@ -190,37 +190,83 @@ jobs: options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} steps: - - name: Run start commands - run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} - - - uses: actions/checkout@v4 - - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: ${{ matrix.java }} - - - name: Run backward compatibility tests - run: | - chown -R 1000:1000 `pwd` - su `id -un 1000` -c "./scripts/bwctest.sh" - - - name: Upload test reports - if: ${{ always() }} - uses: actions/upload-artifact@v4 - continue-on-error: true - with: - name: test-reports-ubuntu-latest-${{ matrix.java }}-bwc - path: | - sql/build/reports/** - ppl/build/reports/** - core/build/reports/** - common/build/reports/** - opensearch/build/reports/** - integ-test/build/reports/** - protocol/build/reports/** - legacy/build/reports/** - plugin/build/reports/** - doctest/build/testclusters/docTestCluster-0/logs/* - integ-test/build/testclusters/*/logs/* + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - uses: actions/checkout@v4 + + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + + - name: Run backward compatibility tests in rolling upgrade + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./scripts/bwctest-rolling-upgrade.sh" + + - name: Upload test reports + if: ${{ always() }} + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: test-reports-ubuntu-latest-${{ matrix.java }}-bwc + path: | + sql/build/reports/** + ppl/build/reports/** + core/build/reports/** + common/build/reports/** + opensearch/build/reports/** + integ-test/build/reports/** + protocol/build/reports/** + legacy/build/reports/** + plugin/build/reports/** + doctest/build/testclusters/docTestCluster-0/logs/* + integ-test/build/testclusters/*/logs/* + + bwc-tests-full-restart: + needs: Get-CI-Image-Tag + runs-on: ubuntu-latest + strategy: + matrix: + java: [21, 24] + container: + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} + + steps: + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - uses: actions/checkout@v4 + + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + + - name: Run backward compatibility tests in full restart + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./scripts/bwctest-full-restart.sh" + + - name: Upload test reports + if: ${{ always() }} + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: test-reports-ubuntu-latest-${{ matrix.java }}-bwc + path: | + sql/build/reports/** + ppl/build/reports/** + core/build/reports/** + common/build/reports/** + opensearch/build/reports/** + integ-test/build/reports/** + protocol/build/reports/** + legacy/build/reports/** + plugin/build/reports/** + doctest/build/testclusters/docTestCluster-0/logs/* + integ-test/build/testclusters/*/logs/* diff --git a/.github/workflows/stalled.yml b/.github/workflows/stalled.yml index 2b10140bbdf..62b85cecd7b 100644 --- a/.github/workflows/stalled.yml +++ b/.github/workflows/stalled.yml @@ -21,8 +21,9 @@ jobs: with: repo-token: ${{ steps.github_app_token.outputs.token }} stale-pr-label: 'stalled' - stale-pr-message: 'This PR is stalled because it has been open for 30 days with no activity.' - days-before-pr-stale: 30 + stale-pr-message: 'This PR is stalled because it has been open for 2 weeks with no activity.' + days-before-pr-stale: 14 days-before-issue-stale: -1 days-before-pr-close: -1 days-before-issue-close: -1 + exempt-draft-pr: true diff --git a/DEVELOPER_GUIDE.rst b/DEVELOPER_GUIDE.rst index 7482d0675d8..7a1fac66d4a 100644 --- a/DEVELOPER_GUIDE.rst +++ b/DEVELOPER_GUIDE.rst @@ -173,7 +173,6 @@ Here are other files and sub-folders that you are likely to touch: - ``build.gradle``: Gradle build script. - ``docs``: documentation for developers and reference manual for users. - ``doc-test``: code that run .rst docs in ``docs`` folder by Python doctest library. -- ``language-grammar``: centralized package for ANTLR grammar files. See `Language Grammar Package`_ for details. Note that other related project code has already merged into this single repository together: @@ -261,6 +260,8 @@ For faster local iterations, skip integration tests. ``./gradlew build -x integT For integration test, you can use ``-Dtests.class`` "UT full path" to run a task individually. For example ``./gradlew :integ-test:integTest -Dtests.class="*QueryIT"``. +If Prometheus isn't available in your environment, you can skip downloading and starting it by adding ``-DignorePrometheus`` (or setting it to any value other than ``false``) to the command. For example ``./gradlew :integ-test:integTest -DignorePrometheus`` bypasses Prometheus setup and excludes Prometheus-specific integration tests, and ``./gradlew :doctest:doctest -DignorePrometheus`` skips the Prometheus-dependent doctest cases. + To run the task above for specific module, you can do ``./gradlew ::task``. For example, only build core module by ``./gradlew :core:build``. Troubleshooting @@ -442,29 +443,3 @@ with an appropriate label `backport ` is merged to main wi PR. For example, if a PR on main needs to be backported to `1.x` branch, add a label `backport 1.x` to the PR and make sure the backport workflow runs on the PR along with other checks. Once this PR is merged to main, the workflow will create a backport PR to the `1.x` branch. - -Language Grammar Package -======================== - -The ``language-grammar`` package serves as a centralized repository for all ANTLR grammar files used throughout the OpenSearch SQL project. This package contains the definitive versions of grammar files for: - -- SQL parsing (``OpenSearchSQLParser.g4``, ``OpenSearchSQLLexer.g4``) -- PPL parsing (``OpenSearchPPLParser.g4``, ``OpenSearchPPLLexer.g4``) -- Legacy SQL parsing (``OpenSearchLegacySqlParser.g4``, ``OpenSearchLegacySqlLexer.g4``) -- Spark SQL extensions (``SparkSqlBase.g4``, ``FlintSparkSqlExtensions.g4``, ``SqlBaseParser.g4``, ``SqlBaseLexer.g4``) - -Purpose -------- - -The language-grammar package enables sharing of grammar files between the main SQL repository and the Spark repository, ensuring consistency and reducing duplication. Once updated, the package automatically triggers CI to upload the new version to Maven Central for consumption by other projects. - -Updating Grammar Files ----------------------- - -When grammar files are modified in their respective modules (``sql/``, ``ppl/``, ``legacy/``, ``async-query-core/``), they must be manually copied to the ``language-grammar/src/main/antlr4/`` directory. - -**Workflow:** - -1. Modify grammar files in their source locations (e.g., ``sql/src/main/antlr/``) -2. Copy updated files to ``language-grammar/src/main/antlr4/`` -3. Commit changes to trigger automatic Maven publication via CI diff --git a/api/README.md b/api/README.md new file mode 100644 index 00000000000..0288b7ad22c --- /dev/null +++ b/api/README.md @@ -0,0 +1,75 @@ +# Unified Query API + +This module provides a high-level integration layer for the Calcite-based query engine, enabling external systems such as Apache Spark or command-line tools to parse and analyze queries without exposing low-level internals. + +## Overview + +The `UnifiedQueryPlanner` serves as the primary entry point for external consumers. It accepts PPL (Piped Processing Language) queries and returns Calcite `RelNode` logical plans as intermediate representation. + +## Usage + +Use the declarative, fluent builder API to initialize the `UnifiedQueryPlanner`. + +```java +UnifiedQueryPlanner planner = UnifiedQueryPlanner.builder() + .language(QueryType.PPL) + .catalog("opensearch", schema) + .defaultNamespace("opensearch") + .cacheMetadata(true) + .build(); + +RelNode plan = planner.plan("source = opensearch.test"); +``` + +## Development & Testing + +A set of unit tests is provided to validate planner behavior. + +To run tests: + +``` +./gradlew :api:test +``` + +## Integration Guide + +This guide walks through how to integrate unified query planner into your application. + +### Step 1: Add Dependency + +The module is currently published as a snapshot to the AWS Sonatype Snapshots repository. To include it as a dependency in your project, add the following to your `pom.xml` or `build.gradle`: + +```xml + + org.opensearch.query + unified-query-api + YOUR_VERSION_HERE + +``` + +### Step 2: Implement a Calcite Schema + +You must implement the Calcite `Schema` interface and register them using the fluent `catalog()` method on the builder. + +```java +public class MySchema extends AbstractSchema { + @Override + protected Map getTableMap() { + return Map.of( + "test_table", + new AbstractTable() { + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) { + return typeFactory.createStructType( + List.of(typeFactory.createSqlType(SqlTypeName.INTEGER)), + List.of("id")); + } + }); + } +} +``` + +## Future Work + +- Expand support to SQL language. +- Extend planner to generate optimized physical plans using Calcite's optimization frameworks. diff --git a/api/build.gradle b/api/build.gradle new file mode 100644 index 00000000000..0b96acabec1 --- /dev/null +++ b/api/build.gradle @@ -0,0 +1,71 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +plugins { + id 'java-library' + id 'jacoco' + id 'com.diffplug.spotless' +} + +dependencies { + api project(':ppl') + + testImplementation group: 'junit', name: 'junit', version: '4.13.2' + testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: "${hamcrest_version}" + testImplementation group: 'org.mockito', name: 'mockito-core', version: "${mockito_version}" + testImplementation group: 'org.apache.calcite', name: 'calcite-testkit', version: '1.41.0' +} + +spotless { + java { + target fileTree('.') { + include '**/*.java' + exclude '**/build/**', '**/build-*/**', 'src/main/gen/**' + } + importOrder() + removeUnusedImports() + trimTrailingWhitespace() + endWithNewline() + googleJavaFormat('1.17.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') + } +} + +test { + testLogging { + events "passed", "skipped", "failed" + exceptionFormat "full" + } +} + +jacocoTestReport { + reports { + html.required = true + xml.required = true + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, + exclude: ['**/antlr/parser/**']) + })) + } +} +test.finalizedBy(project.tasks.jacocoTestReport) +jacocoTestCoverageVerification { + violationRules { + rule { + limit { + minimum = 0.9 + } + + } + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, + exclude: ['**/antlr/parser/**']) + })) + } +} +check.dependsOn jacocoTestCoverageVerification diff --git a/api/src/main/java/org/opensearch/sql/api/EmptyDataSourceService.java b/api/src/main/java/org/opensearch/sql/api/EmptyDataSourceService.java new file mode 100644 index 00000000000..0fa0c38ad3c --- /dev/null +++ b/api/src/main/java/org/opensearch/sql/api/EmptyDataSourceService.java @@ -0,0 +1,51 @@ +package org.opensearch.sql.api; + +import java.util.Map; +import java.util.Set; +import org.opensearch.sql.datasource.DataSourceService; +import org.opensearch.sql.datasource.RequestContext; +import org.opensearch.sql.datasource.model.DataSource; +import org.opensearch.sql.datasource.model.DataSourceMetadata; + +/** A DataSourceService that assumes no access to data sources */ +public class EmptyDataSourceService implements DataSourceService { + public EmptyDataSourceService() {} + + @Override + public DataSource getDataSource(String dataSourceName) { + return null; + } + + @Override + public Set getDataSourceMetadata(boolean isDefaultDataSourceRequired) { + return Set.of(); + } + + @Override + public DataSourceMetadata getDataSourceMetadata(String name) { + return null; + } + + @Override + public void createDataSource(DataSourceMetadata metadata) {} + + @Override + public void updateDataSource(DataSourceMetadata dataSourceMetadata) {} + + @Override + public void patchDataSource(Map dataSourceData) {} + + @Override + public void deleteDataSource(String dataSourceName) {} + + @Override + public Boolean dataSourceExists(String dataSourceName) { + return false; + } + + @Override + public DataSourceMetadata verifyDataSourceAccessAndGetRawMetadata( + String dataSourceName, RequestContext context) { + return null; + } +} diff --git a/api/src/main/java/org/opensearch/sql/api/UnifiedQueryPlanner.java b/api/src/main/java/org/opensearch/sql/api/UnifiedQueryPlanner.java new file mode 100644 index 00000000000..6a524ec307a --- /dev/null +++ b/api/src/main/java/org/opensearch/sql/api/UnifiedQueryPlanner.java @@ -0,0 +1,227 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.api; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import org.antlr.v4.runtime.tree.ParseTree; +import org.apache.calcite.jdbc.CalciteSchema; +import org.apache.calcite.plan.RelTraitDef; +import org.apache.calcite.rel.RelCollation; +import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Sort; +import org.apache.calcite.rel.logical.LogicalSort; +import org.apache.calcite.rel.metadata.DefaultRelMetadataProvider; +import org.apache.calcite.schema.Schema; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.tools.FrameworkConfig; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.Programs; +import org.opensearch.sql.ast.statement.Query; +import org.opensearch.sql.ast.statement.Statement; +import org.opensearch.sql.ast.tree.UnresolvedPlan; +import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.calcite.CalciteRelNodeVisitor; +import org.opensearch.sql.calcite.SysLimit; +import org.opensearch.sql.common.antlr.Parser; +import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.executor.QueryType; +import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; +import org.opensearch.sql.ppl.parser.AstBuilder; +import org.opensearch.sql.ppl.parser.AstStatementBuilder; + +/** + * {@code UnifiedQueryPlanner} provides a high-level API for parsing and analyzing queries using the + * Calcite-based query engine. It serves as the primary integration point for external consumers + * such as Spark or command-line tools, abstracting away Calcite internals. + */ +public class UnifiedQueryPlanner { + /** The type of query language being used (e.g., PPL). */ + private final QueryType queryType; + + /** The parser instance responsible for converting query text into a parse tree. */ + private final Parser parser; + + /** Calcite framework configuration used during logical plan construction. */ + private final FrameworkConfig config; + + /** AST-to-RelNode visitor that builds logical plans from the parsed AST. */ + private final CalciteRelNodeVisitor relNodeVisitor = + new CalciteRelNodeVisitor(new EmptyDataSourceService()); + + /** + * Constructs a UnifiedQueryPlanner for a given query type and schema root. + * + * @param queryType the query language type (e.g., PPL) + * @param rootSchema the root Calcite schema containing all catalogs and tables + * @param defaultPath dot-separated path of schema to set as default schema + */ + public UnifiedQueryPlanner(QueryType queryType, SchemaPlus rootSchema, String defaultPath) { + this.queryType = queryType; + this.parser = buildQueryParser(queryType); + this.config = buildCalciteConfig(rootSchema, defaultPath); + } + + /** + * Parses and analyzes a query string into a Calcite logical plan (RelNode). TODO: Generate + * optimal physical plan to fully unify query execution and leverage Calcite's optimizer. + * + * @param query the raw query string in PPL or other supported syntax + * @return a logical plan representing the query + */ + public RelNode plan(String query) { + try { + return preserveCollation(analyze(parse(query))); + } catch (SyntaxCheckException e) { + // Re-throw syntax error without wrapping + throw e; + } catch (Exception e) { + throw new IllegalStateException("Failed to plan query", e); + } + } + + private Parser buildQueryParser(QueryType queryType) { + if (queryType == QueryType.PPL) { + return new PPLSyntaxParser(); + } + throw new IllegalArgumentException("Unsupported query type: " + queryType); + } + + private FrameworkConfig buildCalciteConfig(SchemaPlus rootSchema, String defaultPath) { + SchemaPlus defaultSchema = findSchemaByPath(rootSchema, defaultPath); + return Frameworks.newConfigBuilder() + .parserConfig(SqlParser.Config.DEFAULT) + .defaultSchema(defaultSchema) + .traitDefs((List) null) + .programs(Programs.calc(DefaultRelMetadataProvider.INSTANCE)) + .build(); + } + + private static SchemaPlus findSchemaByPath(SchemaPlus rootSchema, String defaultPath) { + if (defaultPath == null) { + return rootSchema; + } + + // Find schema by the path recursively + SchemaPlus current = rootSchema; + for (String part : defaultPath.split("\\.")) { + current = current.getSubSchema(part); + if (current == null) { + throw new IllegalArgumentException("Invalid default catalog path: " + defaultPath); + } + } + return current; + } + + private UnresolvedPlan parse(String query) { + ParseTree cst = parser.parse(query); + AstStatementBuilder astStmtBuilder = + new AstStatementBuilder( + new AstBuilder(query), AstStatementBuilder.StatementBuilderContext.builder().build()); + Statement statement = cst.accept(astStmtBuilder); + + if (statement instanceof Query) { + return ((Query) statement).getPlan(); + } + throw new UnsupportedOperationException( + "Only query statements are supported but got " + statement.getClass().getSimpleName()); + } + + private RelNode analyze(UnresolvedPlan ast) { + // TODO: Hardcoded query size limit (10000) for now as only logical plan is generated. + CalcitePlanContext calcitePlanContext = + CalcitePlanContext.create(config, new SysLimit(10000, 10000, 10000), queryType); + return relNodeVisitor.analyze(ast, calcitePlanContext); + } + + private RelNode preserveCollation(RelNode logical) { + RelNode calcitePlan = logical; + RelCollation collation = logical.getTraitSet().getCollation(); + if (!(logical instanceof Sort) && collation != RelCollations.EMPTY) { + calcitePlan = LogicalSort.create(logical, collation, null, null); + } + return calcitePlan; + } + + /** Builder for {@link UnifiedQueryPlanner}, supporting declarative fluent API. */ + public static Builder builder() { + return new Builder(); + } + + /** + * Builder for {@link UnifiedQueryPlanner}, supporting both declarative and dynamic schema + * registration for use in query planning. + */ + public static class Builder { + private final Map catalogs = new HashMap<>(); + private String defaultNamespace; + private QueryType queryType; + private boolean cacheMetadata; + + /** + * Sets the query language frontend to be used by the planner. + * + * @param queryType the {@link QueryType}, such as PPL + * @return this builder instance + */ + public Builder language(QueryType queryType) { + this.queryType = queryType; + return this; + } + + /** + * Registers a catalog with the specified name and its associated schema. The schema can be a + * flat or nested structure (e.g., catalog → schema → table), depending on how data is + * organized. + * + * @param name the name of the catalog to register + * @param schema the schema representing the structure of the catalog + * @return this builder instance + */ + public Builder catalog(String name, Schema schema) { + catalogs.put(name, schema); + return this; + } + + /** + * Sets the default namespace path for resolving unqualified table names. + * + * @param namespace dot-separated path (e.g., "spark_catalog.default" or "opensearch") + * @return this builder instance + */ + public Builder defaultNamespace(String namespace) { + this.defaultNamespace = namespace; + return this; + } + + /** + * Enables or disables catalog metadata caching in the root schema. + * + * @param cache whether to enable metadata caching + * @return this builder instance + */ + public Builder cacheMetadata(boolean cache) { + this.cacheMetadata = cache; + return this; + } + + /** + * Builds a {@link UnifiedQueryPlanner} with the configuration. + * + * @return a new instance of {@link UnifiedQueryPlanner} + */ + public UnifiedQueryPlanner build() { + Objects.requireNonNull(queryType, "Must specify language before build"); + SchemaPlus rootSchema = CalciteSchema.createRootSchema(true, cacheMetadata).plus(); + catalogs.forEach(rootSchema::add); + return new UnifiedQueryPlanner(queryType, rootSchema, defaultNamespace); + } + } +} diff --git a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerTest.java b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerTest.java new file mode 100644 index 00000000000..0f7754ba501 --- /dev/null +++ b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerTest.java @@ -0,0 +1,182 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.api; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; + +import java.util.List; +import java.util.Map; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.schema.Schema; +import org.apache.calcite.schema.Table; +import org.apache.calcite.schema.impl.AbstractSchema; +import org.apache.calcite.schema.impl.AbstractTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.junit.Test; +import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.executor.QueryType; + +public class UnifiedQueryPlannerTest { + + /** Test schema consists of a test table with id and name columns */ + private final AbstractSchema testSchema = + new AbstractSchema() { + @Override + protected Map getTableMap() { + return Map.of( + "index", + new AbstractTable() { + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) { + return typeFactory.createStructType( + List.of( + typeFactory.createSqlType(SqlTypeName.INTEGER), + typeFactory.createSqlType(SqlTypeName.VARCHAR)), + List.of("id", "name")); + } + }); + } + }; + + /** Test catalog consists of test schema above */ + private final AbstractSchema testDeepSchema = + new AbstractSchema() { + @Override + protected Map getSubSchemaMap() { + return Map.of("opensearch", testSchema); + } + }; + + @Test + public void testPPLQueryPlanning() { + UnifiedQueryPlanner planner = + UnifiedQueryPlanner.builder() + .language(QueryType.PPL) + .catalog("opensearch", testSchema) + .build(); + + RelNode plan = planner.plan("source = opensearch.index | eval f = abs(id)"); + assertNotNull("Plan should be created", plan); + } + + @Test + public void testPPLQueryPlanningWithDefaultNamespace() { + UnifiedQueryPlanner planner = + UnifiedQueryPlanner.builder() + .language(QueryType.PPL) + .catalog("opensearch", testSchema) + .defaultNamespace("opensearch") + .build(); + + assertNotNull("Plan should be created", planner.plan("source = opensearch.index")); + assertNotNull("Plan should be created", planner.plan("source = index")); + } + + @Test + public void testPPLQueryPlanningWithDefaultNamespaceMultiLevel() { + UnifiedQueryPlanner planner = + UnifiedQueryPlanner.builder() + .language(QueryType.PPL) + .catalog("catalog", testDeepSchema) + .defaultNamespace("catalog.opensearch") + .build(); + + assertNotNull("Plan should be created", planner.plan("source = catalog.opensearch.index")); + assertNotNull("Plan should be created", planner.plan("source = index")); + + // This is valid in SparkSQL, but Calcite requires "catalog" as the default root schema to + // resolve it + assertThrows(IllegalStateException.class, () -> planner.plan("source = opensearch.index")); + } + + @Test + public void testPPLQueryPlanningWithMultipleCatalogs() { + UnifiedQueryPlanner planner = + UnifiedQueryPlanner.builder() + .language(QueryType.PPL) + .catalog("catalog1", testSchema) + .catalog("catalog2", testSchema) + .build(); + + RelNode plan = + planner.plan("source = catalog1.index | lookup catalog2.index id | eval f = abs(id)"); + assertNotNull("Plan should be created with multiple catalogs", plan); + } + + @Test + public void testPPLQueryPlanningWithMultipleCatalogsAndDefaultNamespace() { + UnifiedQueryPlanner planner = + UnifiedQueryPlanner.builder() + .language(QueryType.PPL) + .catalog("catalog1", testSchema) + .catalog("catalog2", testSchema) + .defaultNamespace("catalog2") + .build(); + + RelNode plan = planner.plan("source = catalog1.index | lookup index id | eval f = abs(id)"); + assertNotNull("Plan should be created with multiple catalogs", plan); + } + + @Test + public void testPPLQueryPlanningWithMetadataCaching() { + UnifiedQueryPlanner planner = + UnifiedQueryPlanner.builder() + .language(QueryType.PPL) + .catalog("opensearch", testSchema) + .cacheMetadata(true) + .build(); + + RelNode plan = planner.plan("source = opensearch.index"); + assertNotNull("Plan should be created", plan); + } + + @Test(expected = NullPointerException.class) + public void testMissingQueryLanguage() { + UnifiedQueryPlanner.builder().catalog("opensearch", testSchema).build(); + } + + @Test(expected = IllegalArgumentException.class) + public void testUnsupportedQueryLanguage() { + UnifiedQueryPlanner.builder() + .language(QueryType.SQL) // only PPL is supported for now + .catalog("opensearch", testSchema) + .build(); + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidDefaultNamespacePath() { + UnifiedQueryPlanner.builder() + .language(QueryType.PPL) + .catalog("opensearch", testSchema) + .defaultNamespace("nonexistent") // nonexistent namespace path + .build(); + } + + @Test(expected = IllegalStateException.class) + public void testUnsupportedStatementType() { + UnifiedQueryPlanner planner = + UnifiedQueryPlanner.builder() + .language(QueryType.PPL) + .catalog("opensearch", testSchema) + .build(); + + planner.plan("explain source = index"); // explain statement + } + + @Test(expected = SyntaxCheckException.class) + public void testPlanPropagatingSyntaxCheckException() { + UnifiedQueryPlanner planner = + UnifiedQueryPlanner.builder() + .language(QueryType.PPL) + .catalog("opensearch", testSchema) + .build(); + + planner.plan("source = index | eval"); // Trigger syntax error from parser + } +} diff --git a/build.gradle b/build.gradle index a7ab36ed7f5..f007cef3d20 100644 --- a/build.gradle +++ b/build.gradle @@ -6,7 +6,7 @@ buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "3.3.2-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "3.3.0-SNAPSHOT") isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") version_tokens = opensearch_version.tokenize('-') @@ -92,8 +92,11 @@ apply plugin: 'opensearch.java-agent' repositories { mavenLocal() mavenCentral() // For Elastic Libs that you can use to get started coding until open OpenSearch libs are available + maven { + url 'https://jitpack.io' + content { includeGroup "com.github.babbel" } + } maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" } - maven { url 'https://jitpack.io' } } spotless { @@ -141,7 +144,10 @@ allprojects { resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}" resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${versions.jackson}" resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson:jackson-bom:${versions.jackson}" resolutionStrategy.force 'com.google.protobuf:protobuf-java:3.25.5' + resolutionStrategy.force 'com.google.protobuf:protobuf-java-util:3.25.5' resolutionStrategy.force 'org.locationtech.jts:jts-core:1.19.0' resolutionStrategy.force 'com.google.errorprone:error_prone_annotations:2.28.0' resolutionStrategy.force 'org.checkerframework:checker-qual:3.43.0' @@ -149,6 +155,13 @@ allprojects { resolutionStrategy.force 'org.apache.commons:commons-text:1.11.0' resolutionStrategy.force 'commons-io:commons-io:2.15.0' resolutionStrategy.force 'org.yaml:snakeyaml:2.2' + resolutionStrategy.force 'org.apache.calcite.avatica:avatica-core:1.26.0' + resolutionStrategy.force 'org.slf4j:slf4j-api:2.0.13' + resolutionStrategy.dependencySubstitution { + substitute module('commons-lang:commons-lang') using module('org.apache.commons:commons-lang3:3.18.0') because 'CVE-2025-48924: commons-lang 2.x vulnerable to StackOverflowError' + } + resolutionStrategy.force 'org.apache.calcite.avatica:avatica-core:1.26.0' + resolutionStrategy.force 'org.slf4j:slf4j-api:2.0.13' } } @@ -156,9 +169,60 @@ subprojects { repositories { mavenLocal() mavenCentral() + maven { + url 'https://jitpack.io' + content { includeGroup "com.github.babbel" } + } maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" } maven { url "https://ci.opensearch.org/ci/dbc/snapshots/lucene/" } - maven { url 'https://jitpack.io' } + } + + // Publish internal modules as Maven artifacts for external use, such as by opensearch-spark and opensearch-cli. + def publishedModules = ['api', 'sql', 'ppl', 'core', 'opensearch', 'common', 'protocol', 'datasources', 'legacy'] + if (publishedModules.contains(name)) { + apply plugin: 'java-library' + apply plugin: 'maven-publish' + + def fullArtifactId = "unified-query-${project.name}" + publishing { + publications { + unifiedQuery(MavenPublication) { + from components.java + groupId = "org.opensearch.query" + artifactId = fullArtifactId + + pom { + name = fullArtifactId + description = "OpenSearch unified query ${project.name} module" + licenses { + license { + name = 'The Apache License, Version 2.0' + url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' + } + } + developers { + developer { + name = 'OpenSearch' + url = 'https://github.com/opensearch-project/sql' + } + } + } + } + } + + repositories { + maven { + name = "Snapshots" + url = "https://ci.opensearch.org/ci/dbc/snapshots/maven/" + url = System.getenv("MAVEN_SNAPSHOTS_S3_REPO") + credentials(AwsCredentials) { + accessKey = System.getenv("AWS_ACCESS_KEY_ID") + secretKey = System.getenv("AWS_SECRET_ACCESS_KEY") + sessionToken = System.getenv("AWS_SESSION_TOKEN") + } + } + } + } } } diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java index 32729696658..7613f5e3e17 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java @@ -33,6 +33,8 @@ public enum Key { PPL_REX_MAX_MATCH_LIMIT("plugins.ppl.rex.max_match.limit"), PPL_VALUES_MAX_LIMIT("plugins.ppl.values.max.limit"), PPL_SYNTAX_LEGACY_PREFERRED("plugins.ppl.syntax.legacy.preferred"), + PPL_SUBSEARCH_MAXOUT("plugins.ppl.subsearch.maxout"), + PPL_JOIN_SUBSEARCH_MAXOUT("plugins.ppl.join.subsearch_maxout"), /** Enable Calcite as execution engine */ CALCITE_ENGINE_ENABLED("plugins.calcite.enabled"), @@ -48,6 +50,8 @@ public enum Key { /** Common Settings for SQL and PPL. */ QUERY_MEMORY_LIMIT("plugins.query.memory_limit"), QUERY_SIZE_LIMIT("plugins.query.size_limit"), + QUERY_BUCKET_SIZE("plugins.query.buckets"), + SEARCH_MAX_BUCKETS("search.max_buckets"), ENCYRPTION_MASTER_KEY("plugins.query.datasources.encryption.masterkey"), DATASOURCES_URI_HOSTS_DENY_LIST("plugins.query.datasources.uri.hosts.denylist"), DATASOURCES_LIMIT("plugins.query.datasources.limit"), diff --git a/core/build.gradle b/core/build.gradle index f7de422bae8..12d75582458 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -58,19 +58,41 @@ dependencies { api group: 'com.google.code.gson', name: 'gson', version: '2.8.9' api group: 'com.tdunning', name: 't-digest', version: '3.3' api "net.minidev:json-smart:${versions.json_smart}" - api('org.apache.calcite:calcite-core:1.38.0') { + api('org.apache.calcite:calcite-core:1.41.0') { exclude group: 'net.minidev', module: 'json-smart' - exclude group: 'commons-lang', module: 'commons-lang' } - api 'org.apache.calcite:calcite-linq4j:1.38.0' + + // Substrait with latest version 0.55.1 and SLF4J exclusions to avoid conflicts + implementation('io.substrait:core:0.67.0') { + exclude group: 'ch.qos.logback', module: 'logback-classic' + exclude group: 'ch.qos.logback', module: 'logback-core' + exclude group: 'org.apache.calcite', module: 'calcite-core' + exclude group: 'org.apache.calcite', module: 'calcite-linq4j' + exclude group: 'com.google.code.gson', module: 'gson' + exclude group: 'org.antlr', module: 'antlr4-runtime' + exclude group: 'org.antlr', module: 'antlr4' + } + implementation('io.substrait:isthmus:0.67.0') { + exclude group: 'ch.qos.logback', module: 'logback-classic' + exclude group: 'ch.qos.logback', module: 'logback-core' + exclude group: 'org.apache.calcite', module: 'calcite-core' + exclude group: 'org.apache.calcite', module: 'calcite-linq4j' + exclude group: 'com.google.code.gson', module: 'gson' + exclude group: 'org.antlr', module: 'antlr4-runtime' + exclude group: 'org.antlr', module: 'antlr4' + } + + api 'org.apache.calcite:calcite-linq4j:1.41.0' api project(':common') implementation "com.github.seancfoley:ipaddress:5.4.2" implementation "com.jayway.jsonpath:json-path:2.9.0" + implementation 'com.google.protobuf:protobuf-java-util:3.25.5' annotationProcessor('org.immutables:value:2.8.8') - compileOnly('org.immutables:value-annotations:2.8.8') + compileOnly 'org.immutables:value-annotations:2.8.8' + compileOnlyApi 'com.google.code.findbugs:jsr305:3.0.2' - testImplementation('org.junit.jupiter:junit-jupiter:5.9.3') + testImplementation 'org.junit.jupiter:junit-jupiter:5.9.3' testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: "${hamcrest_version}" testImplementation group: 'org.mockito', name: 'mockito-core', version: "${mockito_version}" testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: "${mockito_version}" diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index d5c37d405ec..e79c15e5881 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -75,6 +75,7 @@ import org.opensearch.sql.ast.tree.Limit; import org.opensearch.sql.ast.tree.Lookup; import org.opensearch.sql.ast.tree.ML; +import org.opensearch.sql.ast.tree.Multisearch; import org.opensearch.sql.ast.tree.Paginate; import org.opensearch.sql.ast.tree.Parse; import org.opensearch.sql.ast.tree.Patterns; @@ -84,12 +85,14 @@ import org.opensearch.sql.ast.tree.Relation; import org.opensearch.sql.ast.tree.RelationSubquery; import org.opensearch.sql.ast.tree.Rename; +import org.opensearch.sql.ast.tree.Replace; import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Rex; import org.opensearch.sql.ast.tree.SPath; import org.opensearch.sql.ast.tree.Search; import org.opensearch.sql.ast.tree.Sort; import org.opensearch.sql.ast.tree.Sort.SortOption; +import org.opensearch.sql.ast.tree.StreamWindow; import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.Timechart; @@ -378,8 +381,7 @@ public LogicalPlan visitRareTopN(RareTopN node, AnalysisContext context) { fields.forEach( field -> newEnv.define(new Symbol(Namespace.FIELD_NAME, field.toString()), field.type())); - List options = node.getArguments(); - Integer noOfResults = (Integer) options.get(0).getValue().getValue(); + Integer noOfResults = node.getNoOfResults(); return new LogicalRareTopN(child, node.getCommandType(), noOfResults, fields, groupBys); } @@ -747,6 +749,11 @@ public LogicalPlan visitTrendline(Trendline node, AnalysisContext context) { computationsAndTypes.build()); } + @Override + public LogicalPlan visitStreamWindow(StreamWindow node, AnalysisContext context) { + throw getOnlyForCalciteException("Streamstats"); + } + @Override public LogicalPlan visitFlatten(Flatten node, AnalysisContext context) { throw getOnlyForCalciteException("Flatten"); @@ -800,6 +807,11 @@ public LogicalPlan visitCloseCursor(CloseCursor closeCursor, AnalysisContext con return new LogicalCloseCursor(closeCursor.getChild().get(0).accept(this, context)); } + @Override + public LogicalPlan visitReplace(Replace node, AnalysisContext context) { + throw getOnlyForCalciteException("Replace"); + } + @Override public LogicalPlan visitJoin(Join node, AnalysisContext context) { throw getOnlyForCalciteException("Join"); @@ -820,6 +832,11 @@ public LogicalPlan visitAppend(Append node, AnalysisContext context) { throw getOnlyForCalciteException("Append"); } + @Override + public LogicalPlan visitMultisearch(Multisearch node, AnalysisContext context) { + throw getOnlyForCalciteException("Multisearch"); + } + private LogicalSort buildSort( LogicalPlan child, AnalysisContext context, Integer count, List sortFields) { ExpressionReferenceOptimizer optimizer = diff --git a/core/src/main/java/org/opensearch/sql/analysis/SelectExpressionAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/SelectExpressionAnalyzer.java index 5e46cfa629d..ccb0f5a8f44 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/SelectExpressionAnalyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/SelectExpressionAnalyzer.java @@ -34,31 +34,45 @@ */ @RequiredArgsConstructor public class SelectExpressionAnalyzer - extends AbstractNodeVisitor, AnalysisContext> { + extends AbstractNodeVisitor< + List, SelectExpressionAnalyzer.AnalysisContextWithOptimizer> { private final ExpressionAnalyzer expressionAnalyzer; - private ExpressionReferenceOptimizer optimizer; - /** Analyze Select fields. */ public List analyze( List selectList, AnalysisContext analysisContext, ExpressionReferenceOptimizer optimizer) { - this.optimizer = optimizer; + // Create per-request context wrapper to avoid shared mutable state + AnalysisContextWithOptimizer contextWithOptimizer = + new AnalysisContextWithOptimizer(analysisContext, optimizer); ImmutableList.Builder builder = new ImmutableList.Builder<>(); for (UnresolvedExpression unresolvedExpression : selectList) { - builder.addAll(unresolvedExpression.accept(this, analysisContext)); + builder.addAll(unresolvedExpression.accept(this, contextWithOptimizer)); } return builder.build(); } + /** Context wrapper to pass optimizer per-request without shared state */ + static class AnalysisContextWithOptimizer { + final AnalysisContext analysisContext; + final ExpressionReferenceOptimizer optimizer; + + AnalysisContextWithOptimizer( + AnalysisContext analysisContext, ExpressionReferenceOptimizer optimizer) { + this.analysisContext = analysisContext; + this.optimizer = optimizer; + } + } + @Override - public List visitField(Field node, AnalysisContext context) { - return Collections.singletonList(DSL.named(node.accept(expressionAnalyzer, context))); + public List visitField(Field node, AnalysisContextWithOptimizer context) { + return Collections.singletonList( + DSL.named(node.accept(expressionAnalyzer, context.analysisContext))); } @Override - public List visitAlias(Alias node, AnalysisContext context) { + public List visitAlias(Alias node, AnalysisContextWithOptimizer context) { // Expand all nested fields if used in SELECT clause if (node.getDelegated() instanceof NestedAllTupleFields) { return node.getDelegated().accept(this, context); @@ -82,20 +96,23 @@ public List visitAlias(Alias node, AnalysisContext context) { * groupExpr)) * */ - private Expression referenceIfSymbolDefined(Alias expr, AnalysisContext context) { + private Expression referenceIfSymbolDefined(Alias expr, AnalysisContextWithOptimizer context) { UnresolvedExpression delegatedExpr = expr.getDelegated(); // Pass named expression because expression like window function loses full name // (OVER clause) and thus depends on name in alias to be replaced correctly - return optimizer.optimize( + return context.optimizer.optimize( DSL.named( - expr.getName(), delegatedExpr.accept(expressionAnalyzer, context), expr.getAlias()), - context); + expr.getName(), + delegatedExpr.accept(expressionAnalyzer, context.analysisContext), + expr.getAlias()), + context.analysisContext); } @Override - public List visitAllFields(AllFields node, AnalysisContext context) { - TypeEnvironment environment = context.peek(); + public List visitAllFields( + AllFields node, AnalysisContextWithOptimizer context) { + TypeEnvironment environment = context.analysisContext.peek(); Map lookupAllFields = environment.lookupAllFields(Namespace.FIELD_NAME); return lookupAllFields.entrySet().stream() .map( @@ -107,8 +124,8 @@ public List visitAllFields(AllFields node, AnalysisContext cont @Override public List visitNestedAllTupleFields( - NestedAllTupleFields node, AnalysisContext context) { - TypeEnvironment environment = context.peek(); + NestedAllTupleFields node, AnalysisContextWithOptimizer context) { + TypeEnvironment environment = context.analysisContext.peek(); Map lookupAllTupleFields = environment.lookupAllTupleFields(Namespace.FIELD_NAME); environment.resolve(new Symbol(Namespace.FIELD_NAME, node.getPath())); @@ -123,7 +140,7 @@ public List visitNestedAllTupleFields( new Function( "nested", List.of(new QualifiedName(List.of(entry.getKey().split("\\."))))) - .accept(expressionAnalyzer, context); + .accept(expressionAnalyzer, context.analysisContext); return DSL.named("nested(" + entry.getKey() + ")", nestedFunc); }) .collect(Collectors.toList()); @@ -137,10 +154,10 @@ public List visitNestedAllTupleFields( * this. Otherwise, what unqualified() returns will override Alias's name as NamedExpression's * name even though the QualifiedName doesn't have qualifier. */ - private String unqualifiedNameIfFieldOnly(Alias node, AnalysisContext context) { + private String unqualifiedNameIfFieldOnly(Alias node, AnalysisContextWithOptimizer context) { UnresolvedExpression selectItem = node.getDelegated(); if (selectItem instanceof QualifiedName) { - QualifierAnalyzer qualifierAnalyzer = new QualifierAnalyzer(context); + QualifierAnalyzer qualifierAnalyzer = new QualifierAnalyzer(context.analysisContext); return qualifierAnalyzer.unqualified((QualifiedName) selectItem); } return node.getName(); diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index 84f7bdbd4a6..0dd475c5612 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -63,6 +63,7 @@ import org.opensearch.sql.ast.tree.Limit; import org.opensearch.sql.ast.tree.Lookup; import org.opensearch.sql.ast.tree.ML; +import org.opensearch.sql.ast.tree.Multisearch; import org.opensearch.sql.ast.tree.Paginate; import org.opensearch.sql.ast.tree.Parse; import org.opensearch.sql.ast.tree.Patterns; @@ -72,11 +73,13 @@ import org.opensearch.sql.ast.tree.Relation; import org.opensearch.sql.ast.tree.RelationSubquery; import org.opensearch.sql.ast.tree.Rename; +import org.opensearch.sql.ast.tree.Replace; import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Rex; import org.opensearch.sql.ast.tree.SPath; import org.opensearch.sql.ast.tree.Search; import org.opensearch.sql.ast.tree.Sort; +import org.opensearch.sql.ast.tree.StreamWindow; import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.Timechart; @@ -244,6 +247,10 @@ public T visitRename(Rename node, C context) { return visitChildren(node, context); } + public T visitReplace(Replace node, C context) { + return visitChildren(node, context); + } + public T visitEval(Eval node, C context) { return visitChildren(node, context); } @@ -404,6 +411,10 @@ public T visitWindow(Window window, C context) { return visitChildren(window, context); } + public T visitStreamWindow(StreamWindow node, C context) { + return visitChildren(node, context); + } + public T visitJoin(Join node, C context) { return visitChildren(node, context); } @@ -431,4 +442,8 @@ public T visitAppendCol(AppendCol node, C context) { public T visitAppend(Append node, C context) { return visitChildren(node, context); } + + public T visitMultisearch(Multisearch node, C context) { + return visitChildren(node, context); + } } diff --git a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java index cc5b578c645..67cc893c5b0 100644 --- a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java @@ -80,6 +80,7 @@ import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; +import org.opensearch.sql.calcite.plan.OpenSearchConstants; /** Class of static methods to create specific node instances. */ @UtilityClass @@ -491,8 +492,8 @@ public static Span spanFromSpanLengthLiteral( UnresolvedExpression field, Literal spanLengthLiteral) { if (spanLengthLiteral.getType() == DataType.STRING) { String spanText = spanLengthLiteral.getValue().toString(); - String valueStr = spanText.replaceAll("[^0-9]", ""); - String unitStr = spanText.replaceAll("[0-9]", ""); + String valueStr = spanText.replaceAll("[^0-9-]", ""); + String unitStr = spanText.replaceAll("[0-9-]", ""); if (valueStr.isEmpty()) { // No numeric value found, use the literal as-is @@ -500,6 +501,10 @@ public static Span spanFromSpanLengthLiteral( } else { // Parse numeric value and unit Integer value = Integer.parseInt(valueStr); + if (value <= 0) { + throw new IllegalArgumentException( + String.format("Zero or negative time interval not supported: %s", spanText)); + } SpanUnit unit = unitStr.isEmpty() ? SpanUnit.NONE : SpanUnit.of(unitStr); return span(field, intLiteral(value), unit); } @@ -535,8 +540,16 @@ public static RareTopN rareTopN( List noOfResults, List groupList, Field... fields) { - return new RareTopN(input, commandType, noOfResults, Arrays.asList(fields), groupList) - .attach(input); + Integer N = + (Integer) + Argument.ArgumentMap.of(noOfResults) + .getOrDefault("noOfResults", new Literal(10, DataType.INTEGER)) + .getValue(); + List removed = + noOfResults.stream() + .filter(argument -> !argument.getArgName().equals("noOfResults")) + .toList(); + return new RareTopN(commandType, N, removed, Arrays.asList(fields), groupList).attach(input); } public static Limit limit(UnresolvedPlan input, Integer limit, Integer offset) { @@ -596,11 +609,25 @@ public static FillNull fillNull(UnresolvedPlan input, UnresolvedExpression repla return FillNull.ofSameValue(replacement, ImmutableList.of()).attach(input); } + public static FillNull fillNull( + UnresolvedPlan input, UnresolvedExpression replacement, boolean useValueSyntax) { + return FillNull.ofSameValue(replacement, ImmutableList.of(), useValueSyntax).attach(input); + } + public static FillNull fillNull( UnresolvedPlan input, UnresolvedExpression replacement, Field... fields) { return FillNull.ofSameValue(replacement, ImmutableList.copyOf(fields)).attach(input); } + public static FillNull fillNull( + UnresolvedPlan input, + UnresolvedExpression replacement, + boolean useValueSyntax, + Field... fields) { + return FillNull.ofSameValue(replacement, ImmutableList.copyOf(fields), useValueSyntax) + .attach(input); + } + public static FillNull fillNull( UnresolvedPlan input, List> fieldAndReplacements) { ImmutableList.Builder> replacementsBuilder = @@ -699,4 +726,9 @@ public static Bin bin(UnresolvedExpression field, Argument... arguments) { return DefaultBin.builder().field(field).alias(alias).build(); } } + + /** Get a reference to the implicit timestamp field {@code @timestamp} */ + public static Field referImplicitTimestampField() { + return AstDSL.field(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP); + } } diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/AllFields.java b/core/src/main/java/org/opensearch/sql/ast/expression/AllFields.java index 1ac66b287c7..0080cf09b90 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/AllFields.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/AllFields.java @@ -18,7 +18,7 @@ public class AllFields extends UnresolvedExpression { public static final AllFields INSTANCE = new AllFields(); - public AllFields() {} + protected AllFields() {} public static AllFields of() { return INSTANCE; diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/Argument.java b/core/src/main/java/org/opensearch/sql/ast/expression/Argument.java index 0e0e032e22b..607e27b7de9 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/Argument.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/Argument.java @@ -37,6 +37,8 @@ public R accept(AbstractNodeVisitor nodeVisitor, C context) { } /** ArgumentMap is a helper class to get argument value by name. */ + @EqualsAndHashCode + @ToString public static class ArgumentMap { private final Map map; diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/IntervalUnit.java b/core/src/main/java/org/opensearch/sql/ast/expression/IntervalUnit.java index 19e1b07e39b..e2da2eabd7e 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/IntervalUnit.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/IntervalUnit.java @@ -14,8 +14,8 @@ @RequiredArgsConstructor public enum IntervalUnit { UNKNOWN, - MICROSECOND, + MILLISECOND, SECOND, MINUTE, HOUR, diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/QualifiedName.java b/core/src/main/java/org/opensearch/sql/ast/expression/QualifiedName.java index 852b61cfa8a..84fb486702a 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/QualifiedName.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/QualifiedName.java @@ -22,6 +22,7 @@ @Getter @EqualsAndHashCode(callSuper = false) public class QualifiedName extends UnresolvedExpression { + public static final String DELIMITER = "."; private final List parts; public QualifiedName(String name) { @@ -94,7 +95,7 @@ public QualifiedName rest() { } public String toString() { - return String.join(".", this.parts); + return String.join(DELIMITER, this.parts); } @Override diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SpanUnit.java b/core/src/main/java/org/opensearch/sql/ast/expression/SpanUnit.java index aadc94d0b8b..dae11fb9a5c 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SpanUnit.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SpanUnit.java @@ -15,6 +15,8 @@ public enum SpanUnit { UNKNOWN("unknown"), NONE(""), + MICROSECOND("us"), + US("us"), MILLISECOND("ms"), MS("ms"), SECONDS("s"), diff --git a/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java b/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java index d592d7691cf..dd918a886a4 100644 --- a/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java +++ b/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java @@ -39,7 +39,9 @@ public enum ExplainFormat { SIMPLE, STANDARD, EXTENDED, - COST + COST, + /** Formats explain output in yaml format. */ + YAML } public static ExplainFormat format(String format) { diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/FillNull.java b/core/src/main/java/org/opensearch/sql/ast/tree/FillNull.java index a51b2495fbd..43c3a654767 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/FillNull.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/FillNull.java @@ -24,13 +24,18 @@ public class FillNull extends UnresolvedPlan { public static FillNull ofVariousValue(List> replacements) { - return new FillNull(replacements); + return new FillNull(replacements, false); } public static FillNull ofSameValue(UnresolvedExpression replacement, List fieldList) { + return ofSameValue(replacement, fieldList, false); + } + + public static FillNull ofSameValue( + UnresolvedExpression replacement, List fieldList, boolean useValueSyntax) { List> replacementPairs = fieldList.stream().map(f -> Pair.of(f, replacement)).toList(); - FillNull instance = new FillNull(replacementPairs); + FillNull instance = new FillNull(replacementPairs, useValueSyntax); if (replacementPairs.isEmpty()) { // no field specified, the replacement value will be applied to all fields. instance.replacementForAll = Optional.of(replacement); @@ -42,8 +47,14 @@ public static FillNull ofSameValue(UnresolvedExpression replacement, List private final List> replacementPairs; - FillNull(List> replacementPairs) { + // Track if value= syntax was used (added in 3.4). Only needed to distinguish from with...in + // since both apply same value to all fields. using syntax is detected by checking if all + // replacement values are the same. + private final boolean useValueSyntax; + + FillNull(List> replacementPairs, boolean useValueSyntax) { this.replacementPairs = replacementPairs; + this.useValueSyntax = useValueSyntax; } private UnresolvedPlan child; diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Multisearch.java b/core/src/main/java/org/opensearch/sql/ast/tree/Multisearch.java new file mode 100644 index 00000000000..a1acefdd70f --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Multisearch.java @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; + +/** Logical plan node for Multisearch operation. Combines results from multiple search queries. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +public class Multisearch extends UnresolvedPlan { + + private UnresolvedPlan child; + private final List subsearches; + + public Multisearch(List subsearches) { + this.subsearches = subsearches; + } + + @Override + public Multisearch attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + if (this.child == null) { + return ImmutableList.copyOf(subsearches); + } else { + return ImmutableList.builder().add(this.child).addAll(subsearches).build(); + } + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitMultisearch(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/RareTopN.java b/core/src/main/java/org/opensearch/sql/ast/tree/RareTopN.java index 3fd3aa3a2c0..6c543ddc8c3 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/RareTopN.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/RareTopN.java @@ -7,7 +7,6 @@ import com.google.common.collect.ImmutableList; import java.util.List; -import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -24,12 +23,11 @@ @ToString @EqualsAndHashCode(callSuper = false) @RequiredArgsConstructor -@AllArgsConstructor public class RareTopN extends UnresolvedPlan { private UnresolvedPlan child; private final CommandType commandType; - // arguments: noOfResults: Integer, countField: String, showCount: Boolean + private final Integer noOfResults; private final List arguments; private final List fields; private final List groupExprList; @@ -54,4 +52,10 @@ public enum CommandType { TOP, RARE } + + public enum Option { + countField, + showCount, + useNull, + } } diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Replace.java b/core/src/main/java/org/opensearch/sql/ast/tree/Replace.java new file mode 100644 index 00000000000..8b2c18cd56c --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Replace.java @@ -0,0 +1,58 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import java.util.Set; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.jetbrains.annotations.Nullable; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.expression.Field; + +@Getter +@Setter +@ToString +@EqualsAndHashCode(callSuper = false) +public class Replace extends UnresolvedPlan { + private final List replacePairs; + private final Set fieldList; + @Nullable private UnresolvedPlan child; + + /** + * Constructor with multiple pattern/replacement pairs. + * + * @param replacePairs List of pattern/replacement pairs + * @param fieldList Set of fields to apply replacements to + */ + public Replace(List replacePairs, Set fieldList) { + this.replacePairs = replacePairs; + this.fieldList = fieldList; + } + + @Override + public Replace attach(UnresolvedPlan child) { + if (null == this.child) { + this.child = child; + } else { + this.child.attach(child); + } + return this; + } + + @Override + public List getChild() { + return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitReplace(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/ReplacePair.java b/core/src/main/java/org/opensearch/sql/ast/tree/ReplacePair.java new file mode 100644 index 00000000000..e3f3897fdf1 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/ReplacePair.java @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.opensearch.sql.ast.expression.Literal; + +/** A pair of pattern and replacement literals for the Replace command. */ +@Getter +@AllArgsConstructor +@EqualsAndHashCode +@ToString +public class ReplacePair { + private final Literal pattern; + private final Literal replacement; +} diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/SPath.java b/core/src/main/java/org/opensearch/sql/ast/tree/SPath.java index bb891e49cec..89eab6cf166 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/SPath.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/SPath.java @@ -9,6 +9,7 @@ import java.util.List; import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; +import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; import org.checkerframework.checker.nullness.qual.Nullable; @@ -19,6 +20,7 @@ @EqualsAndHashCode(callSuper = false) @RequiredArgsConstructor @AllArgsConstructor +@Getter public class SPath extends UnresolvedPlan { private UnresolvedPlan child; diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/StreamWindow.java b/core/src/main/java/org/opensearch/sql/ast/tree/StreamWindow.java new file mode 100644 index 00000000000..ed7bcf10289 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/StreamWindow.java @@ -0,0 +1,71 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.expression.UnresolvedExpression; + +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +public class StreamWindow extends UnresolvedPlan { + + private final List windowFunctionList; + private final List groupList; + private final boolean current; + private final int window; + private final boolean global; + private final UnresolvedExpression resetBefore; + private final UnresolvedExpression resetAfter; + @ToString.Exclude private UnresolvedPlan child; + + /** StreamWindow Constructor. */ + public StreamWindow( + List windowFunctionList, + List groupList, + boolean current, + int window, + boolean global, + UnresolvedExpression resetBefore, + UnresolvedExpression resetAfter) { + this.windowFunctionList = windowFunctionList; + this.groupList = groupList; + this.current = current; + this.window = window; + this.global = global; + this.resetBefore = resetBefore; + this.resetAfter = resetAfter; + } + + public boolean isCurrent() { + return current; + } + + public boolean isGlobal() { + return global; + } + + @Override + public StreamWindow attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitStreamWindow(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Timechart.java b/core/src/main/java/org/opensearch/sql/ast/tree/Timechart.java index 1a4c154209c..19972358721 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/Timechart.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Timechart.java @@ -5,14 +5,43 @@ package org.opensearch.sql.ast.tree; +import static org.opensearch.sql.ast.dsl.AstDSL.aggregate; +import static org.opensearch.sql.ast.dsl.AstDSL.doubleLiteral; +import static org.opensearch.sql.ast.dsl.AstDSL.eval; +import static org.opensearch.sql.ast.dsl.AstDSL.function; +import static org.opensearch.sql.ast.dsl.AstDSL.stringLiteral; +import static org.opensearch.sql.ast.expression.IntervalUnit.MILLISECOND; +import static org.opensearch.sql.ast.tree.Timechart.PerFunctionRateExprBuilder.sum; +import static org.opensearch.sql.ast.tree.Timechart.PerFunctionRateExprBuilder.timestampadd; +import static org.opensearch.sql.ast.tree.Timechart.PerFunctionRateExprBuilder.timestampdiff; +import static org.opensearch.sql.calcite.plan.OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DIVIDE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUM; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIMESTAMPADD; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIMESTAMPDIFF; + import com.google.common.collect.ImmutableList; import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.ToString; import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.dsl.AstDSL; +import org.opensearch.sql.ast.expression.AggregateFunction; +import org.opensearch.sql.ast.expression.Field; +import org.opensearch.sql.ast.expression.Function; +import org.opensearch.sql.ast.expression.IntervalUnit; +import org.opensearch.sql.ast.expression.Let; +import org.opensearch.sql.ast.expression.Span; +import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.ast.expression.UnresolvedExpression; +import org.opensearch.sql.calcite.utils.PlanUtils; /** AST node represent Timechart operation. */ @Getter @@ -49,8 +78,9 @@ public Timechart useOther(Boolean useOther) { } @Override - public Timechart attach(UnresolvedPlan child) { - return toBuilder().child(child).build(); + public UnresolvedPlan attach(UnresolvedPlan child) { + // Transform after child attached to avoid unintentionally overriding it + return toBuilder().child(child).build().transformPerFunction(); } @Override @@ -62,4 +92,118 @@ public List getChild() { public T accept(AbstractNodeVisitor nodeVisitor, C context) { return nodeVisitor.visitTimechart(this, context); } + + /** + * Transform per function to eval-based post-processing on sum result by timechart. Specifically, + * calculate how many seconds are in the time bucket based on the span option dynamically, then + * divide the aggregated sum value by the number of seconds to get the per-second rate. + * + *

For example, with span=5m per_second(field): per second rate = sum(field) / 300 seconds + * + * @return eval+timechart if per function present, or the original timechart otherwise. + */ + private UnresolvedPlan transformPerFunction() { + Optional perFuncOpt = PerFunction.from(aggregateFunction); + if (perFuncOpt.isEmpty()) { + return this; + } + + PerFunction perFunc = perFuncOpt.get(); + Span span = (Span) this.binExpression; + Field spanStartTime = AstDSL.field(IMPLICIT_FIELD_TIMESTAMP); + Function spanEndTime = timestampadd(span.getUnit(), span.getValue(), spanStartTime); + Function spanMillis = timestampdiff(MILLISECOND, spanStartTime, spanEndTime); + final int SECOND_IN_MILLISECOND = 1000; + return eval( + timechart(AstDSL.alias(perFunc.aggName, sum(perFunc.aggArg))), + let(perFunc.aggName) + .multiply(perFunc.seconds * SECOND_IN_MILLISECOND) + .dividedBy(spanMillis)); + } + + private Timechart timechart(UnresolvedExpression newAggregateFunction) { + return this.toBuilder().aggregateFunction(newAggregateFunction).build(); + } + + @RequiredArgsConstructor + static class PerFunction { + private static final Map UNIT_SECONDS = + Map.of( + "per_second", 1, + "per_minute", 60, + "per_hour", 3600, + "per_day", 86400); + private final String aggName; + private final UnresolvedExpression aggArg; + private final int seconds; + + static Optional from(UnresolvedExpression aggExpr) { + if (!(aggExpr instanceof AggregateFunction)) { + return Optional.empty(); + } + + AggregateFunction aggFunc = (AggregateFunction) aggExpr; + String aggFuncName = aggFunc.getFuncName().toLowerCase(Locale.ROOT); + if (!UNIT_SECONDS.containsKey(aggFuncName)) { + return Optional.empty(); + } + + String aggName = toAggName(aggFunc); + return Optional.of( + new PerFunction(aggName, aggFunc.getField(), UNIT_SECONDS.get(aggFuncName))); + } + + private static String toAggName(AggregateFunction aggFunc) { + String fieldName = + (aggFunc.getField() instanceof Field) + ? ((Field) aggFunc.getField()).getField().toString() + : aggFunc.getField().toString(); + return String.format(Locale.ROOT, "%s(%s)", aggFunc.getFuncName(), fieldName); + } + } + + private PerFunctionRateExprBuilder let(String fieldName) { + return new PerFunctionRateExprBuilder(AstDSL.field(fieldName)); + } + + /** Fluent builder for creating Let expressions with mathematical operations. */ + static class PerFunctionRateExprBuilder { + private final Field field; + private UnresolvedExpression expr; + + PerFunctionRateExprBuilder(Field field) { + this.field = field; + this.expr = field; + } + + PerFunctionRateExprBuilder multiply(Integer multiplier) { + // Promote to double literal to avoid integer division in downstream + this.expr = + function( + MULTIPLY.getName().getFunctionName(), expr, doubleLiteral(multiplier.doubleValue())); + return this; + } + + Let dividedBy(UnresolvedExpression divisor) { + return AstDSL.let(field, function(DIVIDE.getName().getFunctionName(), expr, divisor)); + } + + static UnresolvedExpression sum(UnresolvedExpression field) { + return aggregate(SUM.getName().getFunctionName(), field); + } + + static Function timestampadd( + SpanUnit unit, UnresolvedExpression value, UnresolvedExpression timestampField) { + UnresolvedExpression intervalUnit = + stringLiteral(PlanUtils.spanUnitToIntervalUnit(unit).toString()); + return function( + TIMESTAMPADD.getName().getFunctionName(), intervalUnit, value, timestampField); + } + + static Function timestampdiff( + IntervalUnit unit, UnresolvedExpression start, UnresolvedExpression end) { + return function( + TIMESTAMPDIFF.getName().getFunctionName(), stringLiteral(unit.toString()), start, end); + } + } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java index 47735bc4281..30393cfbd96 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java @@ -16,6 +16,7 @@ import java.util.function.BiFunction; import lombok.Getter; import lombok.Setter; +import org.apache.calcite.rel.RelNode; import org.apache.calcite.rex.RexCorrelVariable; import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexNode; @@ -23,6 +24,7 @@ import org.apache.calcite.tools.RelBuilder; import org.opensearch.sql.ast.expression.UnresolvedExpression; import org.opensearch.sql.calcite.utils.CalciteToolsHelper; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.executor.QueryType; import org.opensearch.sql.expression.function.FunctionProperties; @@ -34,11 +36,15 @@ public class CalcitePlanContext { public final ExtendedRexBuilder rexBuilder; public final FunctionProperties functionProperties; public final QueryType queryType; - public final Integer querySizeLimit; + public final SysLimit sysLimit; /** This thread local variable is only used to skip script encoding in script pushdown. */ public static final ThreadLocal skipEncoding = ThreadLocal.withInitial(() -> false); + /** Thread-local switch that tells whether the current query prefers legacy behavior. */ + private static final ThreadLocal legacyPreferredFlag = + ThreadLocal.withInitial(() -> true); + @Getter @Setter private boolean isResolvingJoinCondition = false; @Getter @Setter private boolean isResolvingSubquery = false; @Getter @Setter private boolean inCoalesceFunction = false; @@ -56,9 +62,12 @@ public class CalcitePlanContext { @Getter public Map rexLambdaRefMap; - private CalcitePlanContext(FrameworkConfig config, Integer querySizeLimit, QueryType queryType) { + // Store the complete RelNode tree for serialization during scan + @Getter @Setter private RelNode completeRelNodeTree; + + private CalcitePlanContext(FrameworkConfig config, SysLimit sysLimit, QueryType queryType) { this.config = config; - this.querySizeLimit = querySizeLimit; + this.sysLimit = sysLimit; this.queryType = queryType; this.connection = CalciteToolsHelper.connect(config, TYPE_FACTORY); this.relBuilder = CalciteToolsHelper.create(config, TYPE_FACTORY, connection); @@ -97,12 +106,33 @@ public Optional peekCorrelVar() { } public CalcitePlanContext clone() { - return new CalcitePlanContext(config, querySizeLimit, queryType); + return new CalcitePlanContext(config, sysLimit, queryType); } public static CalcitePlanContext create( - FrameworkConfig config, Integer querySizeLimit, QueryType queryType) { - return new CalcitePlanContext(config, querySizeLimit, queryType); + FrameworkConfig config, SysLimit sysLimit, QueryType queryType) { + return new CalcitePlanContext(config, sysLimit, queryType); + } + + /** + * Executes {@code action} with the thread-local legacy flag set according to the supplied + * settings. + */ + public static void run(Runnable action, Settings settings) { + Boolean preferred = settings.getSettingValue(Settings.Key.PPL_SYNTAX_LEGACY_PREFERRED); + legacyPreferredFlag.set(preferred); + try { + action.run(); + } finally { + legacyPreferredFlag.remove(); + } + } + + /** + * @return {@code true} when the current planning prefer legacy behavior. + */ + public static boolean isLegacyPreferred() { + return legacyPreferredFlag.get(); } public void putRexLambdaRefMap(Map candidateMap) { diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 1f105f5f229..09ad5d4009a 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -15,9 +15,10 @@ import static org.opensearch.sql.ast.tree.Sort.SortOrder.ASC; import static org.opensearch.sql.ast.tree.Sort.SortOrder.DESC; import static org.opensearch.sql.calcite.utils.PlanUtils.ROW_NUMBER_COLUMN_FOR_DEDUP; -import static org.opensearch.sql.calcite.utils.PlanUtils.ROW_NUMBER_COLUMN_NAME; -import static org.opensearch.sql.calcite.utils.PlanUtils.ROW_NUMBER_COLUMN_NAME_MAIN; -import static org.opensearch.sql.calcite.utils.PlanUtils.ROW_NUMBER_COLUMN_NAME_SUBSEARCH; +import static org.opensearch.sql.calcite.utils.PlanUtils.ROW_NUMBER_COLUMN_FOR_MAIN; +import static org.opensearch.sql.calcite.utils.PlanUtils.ROW_NUMBER_COLUMN_FOR_RARE_TOP; +import static org.opensearch.sql.calcite.utils.PlanUtils.ROW_NUMBER_COLUMN_FOR_STREAMSTATS; +import static org.opensearch.sql.calcite.utils.PlanUtils.ROW_NUMBER_COLUMN_FOR_SUBSEARCH; import static org.opensearch.sql.calcite.utils.PlanUtils.getRelation; import static org.opensearch.sql.calcite.utils.PlanUtils.getRexCall; import static org.opensearch.sql.calcite.utils.PlanUtils.transformPlanToAttachChild; @@ -39,6 +40,7 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.plan.ViewExpanders; import org.apache.calcite.rel.RelNode; @@ -48,6 +50,8 @@ import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rel.logical.LogicalValues; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFamily; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexCorrelVariable; @@ -58,9 +62,10 @@ import org.apache.calcite.rex.RexWindowBounds; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.ArraySqlType; +import org.apache.calcite.sql.type.MapSqlType; import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.calcite.sql.validate.SqlValidatorUtil; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.tools.RelBuilder.AggCall; import org.apache.calcite.util.Holder; @@ -111,6 +116,7 @@ import org.opensearch.sql.ast.tree.Lookup; import org.opensearch.sql.ast.tree.Lookup.OutputStrategy; import org.opensearch.sql.ast.tree.ML; +import org.opensearch.sql.ast.tree.Multisearch; import org.opensearch.sql.ast.tree.Paginate; import org.opensearch.sql.ast.tree.Parse; import org.opensearch.sql.ast.tree.Patterns; @@ -119,11 +125,14 @@ import org.opensearch.sql.ast.tree.Regex; import org.opensearch.sql.ast.tree.Relation; import org.opensearch.sql.ast.tree.Rename; +import org.opensearch.sql.ast.tree.Replace; +import org.opensearch.sql.ast.tree.ReplacePair; import org.opensearch.sql.ast.tree.Rex; import org.opensearch.sql.ast.tree.SPath; import org.opensearch.sql.ast.tree.Search; import org.opensearch.sql.ast.tree.Sort; import org.opensearch.sql.ast.tree.Sort.SortOption; +import org.opensearch.sql.ast.tree.StreamWindow; import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.Trendline; @@ -131,6 +140,8 @@ import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.ast.tree.Window; +import org.opensearch.sql.calcite.plan.LogicalSystemLimit; +import org.opensearch.sql.calcite.plan.LogicalSystemLimit.SystemLimitType; import org.opensearch.sql.calcite.plan.OpenSearchConstants; import org.opensearch.sql.calcite.utils.BinUtils; import org.opensearch.sql.calcite.utils.JoinAndLookupUtils; @@ -276,10 +287,13 @@ public RelNode visitRex(Rex node, CalcitePlanContext context) { "Rex pattern must contain at least one named capture group"); } + // TODO: Once JDK 20+ is supported, consider using Pattern.namedGroups() API for more efficient + // named group handling instead of manual parsing in RegexCommonUtils + List newFields = new ArrayList<>(); List newFieldNames = new ArrayList<>(); - for (int i = 0; i < namedGroups.size(); i++) { + for (String groupName : namedGroups) { RexNode extractCall; if (node.getMaxMatch().isPresent() && node.getMaxMatch().get() > 1) { extractCall = @@ -288,7 +302,7 @@ public RelNode visitRex(Rex node, CalcitePlanContext context) { BuiltinFunctionName.REX_EXTRACT_MULTI, fieldRex, context.rexBuilder.makeLiteral(patternStr), - context.relBuilder.literal(i + 1), + context.rexBuilder.makeLiteral(groupName), context.relBuilder.literal(node.getMaxMatch().get())); } else { extractCall = @@ -297,10 +311,10 @@ public RelNode visitRex(Rex node, CalcitePlanContext context) { BuiltinFunctionName.REX_EXTRACT, fieldRex, context.rexBuilder.makeLiteral(patternStr), - context.relBuilder.literal(i + 1)); + context.rexBuilder.makeLiteral(groupName)); } newFields.add(extractCall); - newFieldNames.add(namedGroups.get(i)); + newFieldNames.add(groupName); } if (node.getOffsetField().isPresent()) { @@ -371,7 +385,10 @@ private RelNode handleAllFieldsProject(Project node, CalcitePlanContext context) "Invalid field exclusion: operation would exclude all fields from the result set"); } AllFields allFields = (AllFields) node.getProjectList().getFirst(); - tryToRemoveNestedFields(context); + if (!(allFields instanceof AllFieldsExcludeMeta)) { + // Should not remove nested fields for AllFieldsExcludeMeta. + tryToRemoveNestedFields(context); + } tryToRemoveMetaFields(context, allFields instanceof AllFieldsExcludeMeta); return context.relBuilder.peek(); } @@ -844,7 +861,7 @@ private void projectPlusOverriding( List originalFieldNames = context.relBuilder.peek().getRowType().getFieldNames(); List toOverrideList = originalFieldNames.stream() - .filter(newNames::contains) + .filter(originalName -> shouldOverrideField(originalName, newNames)) .map(a -> (RexNode) context.relBuilder.field(a)) .toList(); // 1. add the new fields, For example "age0, country0" @@ -864,6 +881,17 @@ private void projectPlusOverriding( context.relBuilder.rename(expectedRenameFields); } + private boolean shouldOverrideField(String originalName, List newNames) { + return newNames.stream() + .anyMatch( + newName -> + // Match exact field names (e.g., "age" == "age") for flat fields + newName.equals(originalName) + // OR match nested paths (e.g., "resource.attributes..." starts with + // "resource.") + || newName.startsWith(originalName + ".")); + } + private List> extractInputRefList(List aggCalls) { return aggCalls.stream() .map(RelBuilder.AggCall::over) @@ -883,7 +911,7 @@ private boolean isCountField(RexCall call) { /** * Resolve the aggregation with trimming unused fields to avoid bugs in {@link - * org.apache.calcite.sql2rel.RelDecorrelator#decorrelateRel(Aggregate, boolean)} + * org.apache.calcite.sql2rel.RelDecorrelator#decorrelateRel(Aggregate, boolean, boolean)} * * @param groupExprList group by expression list * @param aggExprList aggregate expression list @@ -994,12 +1022,56 @@ private Pair, List> aggregateWithTrimming( Pair, List> reResolved = resolveAttributesForAggregation(groupExprList, aggExprList, context); + List intendedGroupKeyAliases = getGroupKeyNamesAfterAggregation(reResolved.getLeft()); context.relBuilder.aggregate( context.relBuilder.groupKey(reResolved.getLeft()), reResolved.getRight()); + // During aggregation, Calcite projects both input dependencies and output group-by fields. + // When names conflict, Calcite adds numeric suffixes (e.g., "value0"). + // Apply explicit renaming to restore the intended aliases. + context.relBuilder.rename(intendedGroupKeyAliases); return Pair.of(reResolved.getLeft(), reResolved.getRight()); } + /** + * Imitates {@code Registrar.registerExpression} of {@link RelBuilder} to derive the output order + * of group-by keys after aggregation. + * + *

The projected input reference comes first, while any other computed expression follows. + */ + private List getGroupKeyNamesAfterAggregation(List nodes) { + List reordered = new ArrayList<>(); + List left = new ArrayList<>(); + for (RexNode n : nodes) { + // The same group-key won't be added twice + if (reordered.contains(n) || left.contains(n)) { + continue; + } + if (isInputRef(n)) { + reordered.add(n); + } else { + left.add(n); + } + } + reordered.addAll(left); + return reordered.stream() + .map(this::extractAliasLiteral) + .flatMap(Optional::stream) + .map(RexLiteral::stringValue) + .toList(); + } + + /** Whether a rex node is an aliased input reference */ + private boolean isInputRef(RexNode node) { + return switch (node.getKind()) { + case AS, DESCENDING, NULLS_FIRST, NULLS_LAST -> { + final List operands = ((RexCall) node).operands; + yield isInputRef(operands.getFirst()); + } + default -> node instanceof RexInputRef; + }; + } + /** * Resolve attributes for aggregation. * @@ -1064,22 +1136,7 @@ public RelNode visitAggregation(Aggregation node, CalcitePlanContext context) { Pair, List> aggregationAttributes = aggregateWithTrimming(groupExprList, aggExprList, context); if (toAddHintsOnAggregate) { - final RelHint statHits = - RelHint.builder("stats_args").hintOption(Argument.BUCKET_NULLABLE, "false").build(); - assert context.relBuilder.peek() instanceof LogicalAggregate - : "Stats hits should be added to LogicalAggregate"; - context.relBuilder.hints(statHits); - context - .relBuilder - .getCluster() - .setHintStrategies( - HintStrategyTable.builder() - .hintStrategy( - "stats_args", - (hint, rel) -> { - return rel instanceof LogicalAggregate; - }) - .build()); + addIgnoreNullBucketHintToAggregate(context); } // schema reordering @@ -1098,7 +1155,7 @@ public RelNode visitAggregation(Aggregation node, CalcitePlanContext context) { aggregationAttributes.getLeft().stream() .map(this::extractAliasLiteral) .flatMap(Optional::stream) - .map(ref -> ((RexLiteral) ref).getValueAs(String.class)) + .map(ref -> ref.getValueAs(String.class)) .map(context.relBuilder::field) .map(f -> (RexNode) f) .toList(); @@ -1134,6 +1191,15 @@ private Optional extractAliasLiteral(RexNode node) { public RelNode visitJoin(Join node, CalcitePlanContext context) { List children = node.getChildren(); children.forEach(c -> analyze(c, context)); + // add join.subsearch_maxout limit to subsearch side, 0 and negative means unlimited. + if (context.sysLimit.joinSubsearchLimit() > 0) { + PlanUtils.replaceTop( + context.relBuilder, + LogicalSystemLimit.create( + SystemLimitType.JOIN_SUBSEARCH_MAXOUT, + context.relBuilder.peek(), + context.relBuilder.literal(context.sysLimit.joinSubsearchLimit()))); + } if (node.getJoinCondition().isEmpty()) { // join-with-field-list grammar List leftColumns = context.relBuilder.peek(1).getRowType().getFieldNames(); @@ -1492,6 +1558,354 @@ public RelNode visitWindow(Window node, CalcitePlanContext context) { return context.relBuilder.peek(); } + /** + * Validates type compatibility between replacement value and field for fillnull operation. Throws + * SemanticCheckException if types are incompatible. + */ + private void validateFillNullTypeCompatibility( + RexNode replacement, RexNode fieldRef, String fieldName) { + RelDataTypeFamily replacementFamily = replacement.getType().getFamily(); + RelDataTypeFamily fieldFamily = fieldRef.getType().getFamily(); + + // Check if the replacement type is compatible with the field type + // Allow NULL type family as it's compatible with any type + if (fieldFamily != replacementFamily + && fieldFamily != SqlTypeFamily.NULL + && replacementFamily != SqlTypeFamily.NULL) { + throw new SemanticCheckException( + String.format( + "fillnull failed: replacement value type %s is not compatible with field '%s' " + + "(type: %s). The replacement value type must match the field type.", + replacement.getType().getSqlTypeName(), + fieldName, + fieldRef.getType().getSqlTypeName())); + } + } + + @Override + public RelNode visitStreamWindow(StreamWindow node, CalcitePlanContext context) { + visitChildren(node, context); + + List groupList = node.getGroupList(); + boolean hasGroup = groupList != null && !groupList.isEmpty(); + boolean hasWindow = node.getWindow() > 0; + boolean hasReset = node.getResetBefore() != null || node.getResetAfter() != null; + + // Local helper column names + final String RESET_BEFORE_FLAG_COL = "__reset_before_flag__"; // flag for reset_before + final String RESET_AFTER_FLAG_COL = "__reset_after_flag__"; // flag for reset_after + final String SEGMENT_ID_COL = "__seg_id__"; // segment id + + // CASE: reset + if (hasReset) { + // 1. Build helper columns: seq, before/after flags, segment_id + RelNode leftWithSeg = buildResetHelperColumns(context, node); + + // 2. Run correlate + aggregate with reset-specific filter and cleanup + return buildStreamWindowJoinPlan( + context, + leftWithSeg, + node, + groupList, + ROW_NUMBER_COLUMN_FOR_STREAMSTATS, + SEGMENT_ID_COL, + new String[] { + ROW_NUMBER_COLUMN_FOR_STREAMSTATS, + RESET_BEFORE_FLAG_COL, + RESET_AFTER_FLAG_COL, + SEGMENT_ID_COL + }); + } + + // CASE: global=true + window>0 + has group + if (node.isGlobal() && hasWindow && hasGroup) { + // 1. Add global sequence column for sliding window + RexNode streamSeq = + context + .relBuilder + .aggregateCall(SqlStdOperatorTable.ROW_NUMBER) + .over() + .rowsTo(RexWindowBounds.CURRENT_ROW) + .as(ROW_NUMBER_COLUMN_FOR_STREAMSTATS); + context.relBuilder.projectPlus(streamSeq); + RelNode left = context.relBuilder.build(); + + // 2. Run correlate + aggregate + return buildStreamWindowJoinPlan( + context, + left, + node, + groupList, + ROW_NUMBER_COLUMN_FOR_STREAMSTATS, + null, + new String[] {ROW_NUMBER_COLUMN_FOR_STREAMSTATS}); + } + + // Default + if (hasGroup) { + // only build sequence when there is by condition + RexNode streamSeq = + context + .relBuilder + .aggregateCall(SqlStdOperatorTable.ROW_NUMBER) + .over() + .rowsTo(RexWindowBounds.CURRENT_ROW) + .as(ROW_NUMBER_COLUMN_FOR_STREAMSTATS); + context.relBuilder.projectPlus(streamSeq); + } + + List overExpressions = + node.getWindowFunctionList().stream().map(w -> rexVisitor.analyze(w, context)).toList(); + context.relBuilder.projectPlus(overExpressions); + + // resort when there is by condition + if (hasGroup) { + context.relBuilder.sort(context.relBuilder.field(ROW_NUMBER_COLUMN_FOR_STREAMSTATS)); + context.relBuilder.projectExcept(context.relBuilder.field(ROW_NUMBER_COLUMN_FOR_STREAMSTATS)); + } + + return context.relBuilder.peek(); + } + + private RelNode buildStreamWindowJoinPlan( + CalcitePlanContext context, + RelNode leftWithHelpers, + StreamWindow node, + List groupList, + String seqCol, + String segmentCol, + String[] helperColsToCleanup) { + + final Holder<@Nullable RexCorrelVariable> v = Holder.empty(); + context.relBuilder.push(leftWithHelpers); + context.relBuilder.variable(v::set); + + context.relBuilder.push(leftWithHelpers); + RexNode rightSeq = context.relBuilder.field(seqCol); + RexNode outerSeq = context.relBuilder.field(v.get(), seqCol); + + RexNode filter; + if (segmentCol != null) { // reset condition + RexNode segRight = context.relBuilder.field(segmentCol); + RexNode segOuter = context.relBuilder.field(v.get(), segmentCol); + RexNode frame = buildResetFrameFilter(context, node, outerSeq, rightSeq, segOuter, segRight); + RexNode group = buildGroupFilter(context, groupList, v.get()); + filter = (group == null) ? frame : context.relBuilder.and(frame, group); + } else { // global + window + by condition + RexNode frame = buildFrameFilter(context, node, outerSeq, rightSeq); + RexNode group = buildGroupFilter(context, groupList, v.get()); + filter = context.relBuilder.and(frame, group); + } + context.relBuilder.filter(filter); + + // aggregate all window functions on right side + List aggCalls = buildAggCallsForWindowFunctions(node.getWindowFunctionList(), context); + context.relBuilder.aggregate(context.relBuilder.groupKey(), aggCalls); + RelNode rightAgg = context.relBuilder.build(); + + // correlate LEFT with RIGHT using seq + group fields + context.relBuilder.push(leftWithHelpers); + context.relBuilder.push(rightAgg); + List requiredLeft = buildRequiredLeft(context, seqCol, groupList); + if (segmentCol != null) { // also require seg_id for reset segmentation equality + requiredLeft = new ArrayList<>(requiredLeft); + requiredLeft.add(context.relBuilder.field(2, 0, segmentCol)); + } + context.relBuilder.correlate(JoinRelType.LEFT, v.get().id, requiredLeft); + + // resort to original order + boolean hasGroup = !groupList.isEmpty(); + // resort when 1. global + window + by condition 2.reset + by condition + if (hasGroup) { + context.relBuilder.sort(context.relBuilder.field(seqCol)); + } + + // cleanup helper columns + List cleanup = new ArrayList<>(); + for (String c : helperColsToCleanup) { + cleanup.add(context.relBuilder.field(c)); + } + context.relBuilder.projectExcept(cleanup); + return context.relBuilder.peek(); + } + + private RelNode buildResetHelperColumns(CalcitePlanContext context, StreamWindow node) { + // 1. global sequence to define order + RexNode rowNum = + context + .relBuilder + .aggregateCall(SqlStdOperatorTable.ROW_NUMBER) + .over() + .rowsTo(RexWindowBounds.CURRENT_ROW) + .as(ROW_NUMBER_COLUMN_FOR_STREAMSTATS); + context.relBuilder.projectPlus(rowNum); + + // 2. before/after flags + RexNode beforePred = + (node.getResetBefore() == null) + ? context.relBuilder.literal(false) + : rexVisitor.analyze(node.getResetBefore(), context); + RexNode afterPred = + (node.getResetAfter() == null) + ? context.relBuilder.literal(false) + : rexVisitor.analyze(node.getResetAfter(), context); + RexNode beforeFlag = + context.relBuilder.call( + SqlStdOperatorTable.CASE, + beforePred, + context.relBuilder.literal(1), + context.relBuilder.literal(0)); + RexNode afterFlag = + context.relBuilder.call( + SqlStdOperatorTable.CASE, + afterPred, + context.relBuilder.literal(1), + context.relBuilder.literal(0)); + context.relBuilder.projectPlus(context.relBuilder.alias(beforeFlag, "__reset_before_flag__")); + context.relBuilder.projectPlus(context.relBuilder.alias(afterFlag, "__reset_after_flag__")); + + // 3. session id = SUM(beforeFlag) over (to current) + SUM(afterFlag) over (to 1 preceding) + RexNode sumBefore = + context + .relBuilder + .aggregateCall( + SqlStdOperatorTable.SUM, context.relBuilder.field("__reset_before_flag__")) + .over() + .rowsTo(RexWindowBounds.CURRENT_ROW) + .toRex(); + RexNode sumAfterPrev = + context + .relBuilder + .aggregateCall( + SqlStdOperatorTable.SUM, context.relBuilder.field("__reset_after_flag__")) + .over() + .rowsBetween( + RexWindowBounds.UNBOUNDED_PRECEDING, + RexWindowBounds.preceding(context.relBuilder.literal(1))) + .toRex(); + sumBefore = + context.relBuilder.call( + SqlStdOperatorTable.COALESCE, sumBefore, context.relBuilder.literal(0)); + sumAfterPrev = + context.relBuilder.call( + SqlStdOperatorTable.COALESCE, sumAfterPrev, context.relBuilder.literal(0)); + + RexNode segId = context.relBuilder.call(SqlStdOperatorTable.PLUS, sumBefore, sumAfterPrev); + context.relBuilder.projectPlus(context.relBuilder.alias(segId, "__seg_id__")); + return context.relBuilder.build(); + } + + private RexNode buildFrameFilter( + CalcitePlanContext context, StreamWindow node, RexNode outerSeq, RexNode rightSeq) { + // window always >0 + // frame: either [outer-(w-1), outer] or [outer-w, outer-1] + if (node.isCurrent()) { + RexNode lower = + context.relBuilder.call( + SqlStdOperatorTable.MINUS, + outerSeq, + context.relBuilder.literal(node.getWindow() - 1)); + return context.relBuilder.between(rightSeq, lower, outerSeq); + } else { + RexNode lower = + context.relBuilder.call( + SqlStdOperatorTable.MINUS, outerSeq, context.relBuilder.literal(node.getWindow())); + RexNode upper = + context.relBuilder.call( + SqlStdOperatorTable.MINUS, outerSeq, context.relBuilder.literal(1)); + return context.relBuilder.between(rightSeq, lower, upper); + } + } + + private RexNode buildResetFrameFilter( + CalcitePlanContext context, + StreamWindow node, + RexNode outerSeq, + RexNode rightSeq, + RexNode segIdOuter, + RexNode segIdRight) { + // 1. Compute sequence range (handle running window semantics when window == 0) + RexNode seqFilter; + if (node.getWindow() == 0) { + // running: current => rightSeq <= outerSeq; excluding current => rightSeq < outerSeq + seqFilter = + node.isCurrent() + ? context.relBuilder.lessThanOrEqual(rightSeq, outerSeq) + : context.relBuilder.lessThan(rightSeq, outerSeq); + } else { + // Reuse normal frame filter logic when window > 0 + seqFilter = buildFrameFilter(context, node, outerSeq, rightSeq); + } + // 2. Ensure same segment (seg_id) for reset partitioning + RexNode segFilter = context.relBuilder.equals(segIdRight, segIdOuter); + // 3. Combine filters + return context.relBuilder.and(seqFilter, segFilter); + } + + private RexNode buildGroupFilter( + CalcitePlanContext context, List groupList, RexCorrelVariable correl) { + // build conjunctive equality filters: right.g_i = outer.g_i + if (groupList.isEmpty()) { + return null; + } + List equalsList = + groupList.stream() + .map( + expr -> { + String groupName = extractGroupFieldName(expr); + RexNode rightGroup = context.relBuilder.field(groupName); + RexNode outerGroup = context.relBuilder.field(correl, groupName); + return context.relBuilder.equals(rightGroup, outerGroup); + }) + .toList(); + return context.relBuilder.and(equalsList); + } + + private String extractGroupFieldName(UnresolvedExpression groupExpr) { + if (groupExpr instanceof Alias groupAlias + && groupAlias.getDelegated() instanceof Field groupField) { + return groupField.getField().toString(); + } else if (groupExpr instanceof Field groupField) { + return groupField.getField().toString(); + } else { + throw new IllegalArgumentException( + "Unsupported group expression: only field or alias(field) is supported"); + } + } + + private List buildAggCallsForWindowFunctions( + List windowExprs, CalcitePlanContext context) { + List aggCalls = new ArrayList<>(); + for (UnresolvedExpression expr : windowExprs) { + if (expr instanceof Alias a && a.getDelegated() instanceof WindowFunction wf) { + Function func = (Function) wf.getFunction(); + List args = func.getFuncArgs(); + // first argument is the input field, others are function params + UnresolvedExpression field = args.isEmpty() ? null : args.get(0); + List rest = + args.size() <= 1 ? List.of() : args.subList(1, args.size()); + AggregateFunction aggFunc = new AggregateFunction(func.getFuncName(), field, rest); + AggCall call = aggVisitor.analyze(new Alias(a.getName(), aggFunc), context); + aggCalls.add(call); + } else { + throw new IllegalArgumentException("Unsupported window function in streamstats"); + } + } + return aggCalls; + } + + private List buildRequiredLeft( + CalcitePlanContext context, String seqCol, List groupList) { + List requiredLeft = new ArrayList<>(); + // reference to left seq column + requiredLeft.add(context.relBuilder.field(2, 0, seqCol)); + for (UnresolvedExpression groupExpr : groupList) { + String groupName = extractGroupFieldName(groupExpr); + requiredLeft.add(context.relBuilder.field(2, 0, groupName)); + } + return requiredLeft; + } + @Override public RelNode visitFillNull(FillNull node, CalcitePlanContext context) { visitChildren(node, context); @@ -1500,6 +1914,19 @@ public RelNode visitFillNull(FillNull node, CalcitePlanContext context) { .size()) { throw new IllegalArgumentException("The field list cannot be duplicated in fillnull"); } + + // Validate type compatibility when replacementForAll is present + if (node.getReplacementForAll().isPresent()) { + List fieldsList = context.relBuilder.peek().getRowType().getFieldList(); + RexNode replacement = rexVisitor.analyze(node.getReplacementForAll().get(), context); + + // Validate all fields are compatible with the replacement value + for (RelDataTypeField field : fieldsList) { + RexNode fieldRef = context.rexBuilder.makeInputRef(field.getType(), field.getIndex()); + validateFillNullTypeCompatibility(replacement, fieldRef, field.getName()); + } + } + List projects = new ArrayList<>(); List fieldsList = context.relBuilder.peek().getRowType().getFieldList(); for (RelDataTypeField field : fieldsList) { @@ -1508,6 +1935,8 @@ public RelNode visitFillNull(FillNull node, CalcitePlanContext context) { for (Pair pair : node.getReplacementPairs()) { if (field.getName().equalsIgnoreCase(pair.getLeft().getField().toString())) { RexNode replacement = rexVisitor.analyze(pair.getRight(), context); + // Validate type compatibility before COALESCE + validateFillNullTypeCompatibility(replacement, fieldRef, field.getName()); RexNode coalesce = context.rexBuilder.coalesce(fieldRef, replacement); RexNode coalesceWithAlias = context.relBuilder.alias(coalesce, field.getName()); projects.add(coalesceWithAlias); @@ -1543,7 +1972,7 @@ public RelNode visitAppendCol(AppendCol node, CalcitePlanContext context) { List.of(), WindowFrame.toCurrentRow()); context.relBuilder.projectPlus( - context.relBuilder.alias(mainRowNumber, ROW_NUMBER_COLUMN_NAME_MAIN)); + context.relBuilder.alias(mainRowNumber, ROW_NUMBER_COLUMN_FOR_MAIN)); // 3. build subsearch tree (attach relation to subsearch) UnresolvedPlan relation = getRelation(node); @@ -1561,7 +1990,7 @@ public RelNode visitAppendCol(AppendCol node, CalcitePlanContext context) { List.of(), WindowFrame.toCurrentRow()); context.relBuilder.projectPlus( - context.relBuilder.alias(subsearchRowNumber, ROW_NUMBER_COLUMN_NAME_SUBSEARCH)); + context.relBuilder.alias(subsearchRowNumber, ROW_NUMBER_COLUMN_FOR_SUBSEARCH)); List subsearchFields = context.relBuilder.peek().getRowType().getFieldNames(); List mainFields = context.relBuilder.peek(1).getRowType().getFieldNames(); @@ -1575,8 +2004,8 @@ public RelNode visitAppendCol(AppendCol node, CalcitePlanContext context) { // 7. join with condition `_row_number_main_ = _row_number_subsearch_` RexNode joinCondition = context.relBuilder.equals( - context.relBuilder.field(2, 0, ROW_NUMBER_COLUMN_NAME_MAIN), - context.relBuilder.field(2, 1, ROW_NUMBER_COLUMN_NAME_SUBSEARCH)); + context.relBuilder.field(2, 0, ROW_NUMBER_COLUMN_FOR_MAIN), + context.relBuilder.field(2, 1, ROW_NUMBER_COLUMN_FOR_SUBSEARCH)); context.relBuilder.join( JoinAndLookupUtils.translateJoinType(Join.JoinType.FULL), joinCondition); @@ -1584,8 +2013,8 @@ public RelNode visitAppendCol(AppendCol node, CalcitePlanContext context) { // 8. if override = false, drop both _row_number_ columns context.relBuilder.projectExcept( List.of( - context.relBuilder.field(ROW_NUMBER_COLUMN_NAME_MAIN), - context.relBuilder.field(ROW_NUMBER_COLUMN_NAME_SUBSEARCH))); + context.relBuilder.field(ROW_NUMBER_COLUMN_FOR_MAIN), + context.relBuilder.field(ROW_NUMBER_COLUMN_FOR_SUBSEARCH))); return context.relBuilder.peek(); } else { // 9. if override = true, override the duplicated columns in main by subsearch values @@ -1597,11 +2026,11 @@ public RelNode visitAppendCol(AppendCol node, CalcitePlanContext context) { mainFields.stream().filter(subsearchFields::contains).collect(Collectors.toSet()); RexNode caseCondition = context.relBuilder.equals( - context.relBuilder.field(ROW_NUMBER_COLUMN_NAME_MAIN), - context.relBuilder.field(ROW_NUMBER_COLUMN_NAME_SUBSEARCH)); + context.relBuilder.field(ROW_NUMBER_COLUMN_FOR_MAIN), + context.relBuilder.field(ROW_NUMBER_COLUMN_FOR_SUBSEARCH)); for (int mainFieldIndex = 0; mainFieldIndex < mainFields.size(); mainFieldIndex++) { String mainFieldName = mainFields.get(mainFieldIndex); - if (mainFieldName.equals(ROW_NUMBER_COLUMN_NAME_MAIN)) { + if (mainFieldName.equals(ROW_NUMBER_COLUMN_FOR_MAIN)) { continue; } finalFieldNames.add(mainFieldName); @@ -1626,7 +2055,7 @@ public RelNode visitAppendCol(AppendCol node, CalcitePlanContext context) { subsearchFieldIndex < subsearchFields.size(); subsearchFieldIndex++) { String subsearchFieldName = subsearchFields.get(subsearchFieldIndex); - if (subsearchFieldName.equals(ROW_NUMBER_COLUMN_NAME_SUBSEARCH)) { + if (subsearchFieldName.equals(ROW_NUMBER_COLUMN_FOR_SUBSEARCH)) { continue; } if (!duplicatedFields.contains(subsearchFieldName)) { @@ -1649,65 +2078,71 @@ public RelNode visitAppend(Append node, CalcitePlanContext context) { node.getSubSearch().accept(new EmptySourcePropagateVisitor(), null); prunedSubSearch.accept(this, context); - // 3. Merge two query schemas + // 3. Merge two query schemas using shared logic RelNode subsearchNode = context.relBuilder.build(); RelNode mainNode = context.relBuilder.build(); - List mainFields = mainNode.getRowType().getFieldList(); - List subsearchFields = subsearchNode.getRowType().getFieldList(); - Map subsearchFieldMap = - subsearchFields.stream() - .map(typeField -> Pair.of(typeField.getName(), typeField)) - .collect(Collectors.toMap(Pair::getKey, Pair::getValue)); - boolean[] isSelected = new boolean[subsearchFields.size()]; - List names = new ArrayList<>(); - List mainUnionProjects = new ArrayList<>(); - List subsearchUnionProjects = new ArrayList<>(); - - // 3.1 Start with main query's schema. If subsearch plan doesn't have matched column, - // add same type column in place with NULL literal - for (int i = 0; i < mainFields.size(); i++) { - mainUnionProjects.add(context.rexBuilder.makeInputRef(mainNode, i)); - RelDataTypeField mainField = mainFields.get(i); - RelDataTypeField subsearchField = subsearchFieldMap.get(mainField.getName()); - names.add(mainField.getName()); - if (subsearchFieldMap.containsKey(mainField.getName()) - && subsearchField != null - && subsearchField.getType().equals(mainField.getType())) { - subsearchUnionProjects.add( - context.rexBuilder.makeInputRef(subsearchNode, subsearchField.getIndex())); - isSelected[subsearchField.getIndex()] = true; - } else { - subsearchUnionProjects.add(context.rexBuilder.makeNullLiteral(mainField.getType())); - } + + // Use shared schema merging logic that handles type conflicts via field renaming + List nodesToMerge = Arrays.asList(mainNode, subsearchNode); + List projectedNodes = + SchemaUnifier.buildUnifiedSchemaWithConflictResolution(nodesToMerge, context); + + // 4. Union the projected plans + for (RelNode projectedNode : projectedNodes) { + context.relBuilder.push(projectedNode); } + context.relBuilder.union(true); + return context.relBuilder.peek(); + } - // 3.2 Add remaining subsearch columns to the merged schema - for (int j = 0; j < subsearchFields.size(); j++) { - RelDataTypeField subsearchField = subsearchFields.get(j); - if (!isSelected[j]) { - mainUnionProjects.add(context.rexBuilder.makeNullLiteral(subsearchField.getType())); - subsearchUnionProjects.add(context.rexBuilder.makeInputRef(subsearchNode, j)); - names.add(subsearchField.getName()); + @Override + public RelNode visitMultisearch(Multisearch node, CalcitePlanContext context) { + List subsearchNodes = new ArrayList<>(); + for (UnresolvedPlan subsearch : node.getSubsearches()) { + UnresolvedPlan prunedSubSearch = subsearch.accept(new EmptySourcePropagateVisitor(), null); + prunedSubSearch.accept(this, context); + subsearchNodes.add(context.relBuilder.build()); + } + + // Use shared schema merging logic that handles type conflicts via field renaming + List alignedNodes = + SchemaUnifier.buildUnifiedSchemaWithConflictResolution(subsearchNodes, context); + + for (RelNode alignedNode : alignedNodes) { + context.relBuilder.push(alignedNode); + } + context.relBuilder.union(true, alignedNodes.size()); + + RelDataType rowType = context.relBuilder.peek().getRowType(); + String timestampField = findTimestampField(rowType); + if (timestampField != null) { + RelDataTypeField timestampFieldRef = rowType.getField(timestampField, false, false); + if (timestampFieldRef != null) { + RexNode timestampRef = + context.rexBuilder.makeInputRef( + context.relBuilder.peek(), timestampFieldRef.getIndex()); + context.relBuilder.sort(context.relBuilder.desc(timestampRef)); } } - // 3.3 Uniquify names in case the merged names have duplicates - List uniqNames = - SqlValidatorUtil.uniquify(names, SqlValidatorUtil.EXPR_SUGGESTER, true); - - // 4. Apply new schema over two query plans - RelNode projectedMainNode = - context.relBuilder.push(mainNode).project(mainUnionProjects, uniqNames).build(); - RelNode projectedSubsearchNode = - context.relBuilder.push(subsearchNode).project(subsearchUnionProjects, uniqNames).build(); - - // 5. Union all two projected plans - context.relBuilder.push(projectedMainNode); - context.relBuilder.push(projectedSubsearchNode); - context.relBuilder.union(true); return context.relBuilder.peek(); } + /** + * Finds the @timestamp field for multisearch ordering. Only @timestamp field is used for + * timestamp interleaving. Other timestamp-like fields are ignored. + * + * @param rowType The row type to search for @timestamp field + * @return "@timestamp" if the field exists, or null if not found + */ + private String findTimestampField(RelDataType rowType) { + RelDataTypeField field = rowType.getField("@timestamp", false, false); + if (field != null) { + return "@timestamp"; + } + return null; + } + /* * Unsupported Commands of PPL with Calcite for OpenSearch 3.0.0-beta */ @@ -1744,9 +2179,8 @@ public RelNode visitKmeans(Kmeans node, CalcitePlanContext context) { @Override public RelNode visitRareTopN(RareTopN node, CalcitePlanContext context) { visitChildren(node, context); - - ArgumentMap arguments = ArgumentMap.of(node.getArguments()); - String countFieldName = (String) arguments.get("countField").getValue(); + ArgumentMap argumentMap = ArgumentMap.of(node.getArguments()); + String countFieldName = (String) argumentMap.get(RareTopN.Option.countField.name()).getValue(); if (context.relBuilder.peek().getRowType().getFieldNames().contains(countFieldName)) { throw new IllegalArgumentException( "Field `" @@ -1761,9 +2195,28 @@ public RelNode visitRareTopN(RareTopN node, CalcitePlanContext context) { groupExprList.addAll(fieldList); List aggExprList = List.of(AstDSL.alias(countFieldName, AstDSL.aggregate("count", null))); + + // if usenull=false, add a isNotNull before Aggregate and the hint to this Aggregate + Boolean bucketNullable = (Boolean) argumentMap.get(RareTopN.Option.useNull.name()).getValue(); + boolean toAddHintsOnAggregate = false; + if (!bucketNullable && !groupExprList.isEmpty()) { + toAddHintsOnAggregate = true; + // add isNotNull filter before aggregation to filter out null bucket + List groupByList = + groupExprList.stream().map(expr -> rexVisitor.analyze(expr, context)).toList(); + context.relBuilder.filter( + PlanUtils.getSelectColumns(groupByList).stream() + .map(context.relBuilder::field) + .map(context.relBuilder::isNotNull) + .toList()); + } aggregateWithTrimming(groupExprList, aggExprList, context); - // 2. add a window column + if (toAddHintsOnAggregate) { + addIgnoreNullBucketHintToAggregate(context); + } + + // 2. add count() column with sort direction List partitionKeys = rexVisitor.analyze(node.getGroupExprList(), context); RexNode countField; if (node.getCommandType() == RareTopN.CommandType.TOP) { @@ -1771,6 +2224,7 @@ public RelNode visitRareTopN(RareTopN node, CalcitePlanContext context) { } else { countField = context.relBuilder.field(countFieldName); } + RexNode rowNumberWindowOver = PlanUtils.makeOver( context, @@ -1781,26 +2235,46 @@ public RelNode visitRareTopN(RareTopN node, CalcitePlanContext context) { List.of(countField), WindowFrame.toCurrentRow()); context.relBuilder.projectPlus( - context.relBuilder.alias(rowNumberWindowOver, ROW_NUMBER_COLUMN_NAME)); + context.relBuilder.alias(rowNumberWindowOver, ROW_NUMBER_COLUMN_FOR_RARE_TOP)); // 3. filter row_number() <= k in each partition - Integer N = (Integer) arguments.get("noOfResults").getValue(); + int k = node.getNoOfResults(); context.relBuilder.filter( context.relBuilder.lessThanOrEqual( - context.relBuilder.field(ROW_NUMBER_COLUMN_NAME), context.relBuilder.literal(N))); + context.relBuilder.field(ROW_NUMBER_COLUMN_FOR_RARE_TOP), + context.relBuilder.literal(k))); // 4. project final output. the default output is group by list + field list - Boolean showCount = (Boolean) arguments.get("showCount").getValue(); + Boolean showCount = (Boolean) argumentMap.get(RareTopN.Option.showCount.name()).getValue(); if (showCount) { - context.relBuilder.projectExcept(context.relBuilder.field(ROW_NUMBER_COLUMN_NAME)); + context.relBuilder.projectExcept(context.relBuilder.field(ROW_NUMBER_COLUMN_FOR_RARE_TOP)); } else { context.relBuilder.projectExcept( - context.relBuilder.field(ROW_NUMBER_COLUMN_NAME), + context.relBuilder.field(ROW_NUMBER_COLUMN_FOR_RARE_TOP), context.relBuilder.field(countFieldName)); } return context.relBuilder.peek(); } + private static void addIgnoreNullBucketHintToAggregate(CalcitePlanContext context) { + final RelHint statHits = + RelHint.builder("stats_args").hintOption(Argument.BUCKET_NULLABLE, "false").build(); + assert context.relBuilder.peek() instanceof LogicalAggregate + : "Stats hits should be added to LogicalAggregate"; + context.relBuilder.hints(statHits); + context + .relBuilder + .getCluster() + .setHintStrategies( + HintStrategyTable.builder() + .hintStrategy( + "stats_args", + (hint, rel) -> { + return rel instanceof LogicalAggregate; + }) + .build()); + } + @Override public RelNode visitTableFunction(TableFunction node, CalcitePlanContext context) { throw new CalciteUnsupportedException("Table function is unsupported in Calcite"); @@ -1865,6 +2339,9 @@ public RelNode visitFlatten(Flatten node, CalcitePlanContext context) { /** Helper method to get the function name for proper column naming */ private String getValueFunctionName(UnresolvedExpression aggregateFunction) { + if (aggregateFunction instanceof Alias) { + return ((Alias) aggregateFunction).getName(); + } if (!(aggregateFunction instanceof AggregateFunction)) { return "value"; } @@ -2348,6 +2825,81 @@ public RelNode visitValues(Values values, CalcitePlanContext context) { } } + @Override + public RelNode visitReplace(Replace node, CalcitePlanContext context) { + visitChildren(node, context); + + List fieldNames = context.relBuilder.peek().getRowType().getFieldNames(); + + // Create a set of field names to replace for quick lookup + Set fieldsToReplace = + node.getFieldList().stream().map(f -> f.getField().toString()).collect(Collectors.toSet()); + + // Validate that all fields to replace exist by calling field() on each + // This leverages relBuilder.field()'s built-in validation which throws + // IllegalArgumentException if any field doesn't exist + for (String fieldToReplace : fieldsToReplace) { + context.relBuilder.field(fieldToReplace); + } + + List projectList = new ArrayList<>(); + + // Project all fields, replacing specified ones in-place + for (String fieldName : fieldNames) { + if (fieldsToReplace.contains(fieldName)) { + // Replace this field in-place with all pattern/replacement pairs applied sequentially + RexNode fieldRef = context.relBuilder.field(fieldName); + + // Apply all replacement pairs sequentially (nested REPLACE calls) + for (ReplacePair pair : node.getReplacePairs()) { + RexNode patternNode = rexVisitor.analyze(pair.getPattern(), context); + RexNode replacementNode = rexVisitor.analyze(pair.getReplacement(), context); + + String patternStr = pair.getPattern().getValue().toString(); + String replacementStr = pair.getReplacement().getValue().toString(); + + if (patternStr.contains("*")) { + WildcardUtils.validateWildcardSymmetry(patternStr, replacementStr); + + String regexPattern = WildcardUtils.convertWildcardPatternToRegex(patternStr); + String regexReplacement = + WildcardUtils.convertWildcardReplacementToRegex(replacementStr); + + RexNode regexPatternNode = + context.rexBuilder.makeLiteral( + regexPattern, + context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR), + true); + RexNode regexReplacementNode = + context.rexBuilder.makeLiteral( + regexReplacement, + context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR), + true); + + fieldRef = + context.rexBuilder.makeCall( + org.apache.calcite.sql.fun.SqlLibraryOperators.REGEXP_REPLACE_3, + fieldRef, + regexPatternNode, + regexReplacementNode); + } else { + fieldRef = + context.relBuilder.call( + SqlStdOperatorTable.REPLACE, fieldRef, patternNode, replacementNode); + } + } + + projectList.add(fieldRef); + } else { + // Keep original field unchanged + projectList.add(context.relBuilder.field(fieldName)); + } + } + + context.relBuilder.project(projectList, fieldNames); + return context.relBuilder.peek(); + } + private void buildParseRelNode(Parse node, CalcitePlanContext context) { RexNode sourceField = rexVisitor.analyze(node.getSourceField(), context); ParseMethod parseMethod = node.getParseMethod(); @@ -2421,6 +2973,21 @@ private void buildParseRelNode(Parse node, CalcitePlanContext context) { projectPlusOverriding(newFields, groupCandidates, context); } + /** + * CALCITE-6981 introduced a stricter type checking for Array type in {@link RexToLixTranslator}. + * We defined a MAP(VARCHAR, ANY) in {@link UserDefinedFunctionUtils#nullablePatternAggList}, when + * we convert the value type to ArraySqlType, it will check the source data type by {@link + * RelDataType#getComponentType()} which will return null due to the source type is ANY. + */ + private RexNode explicitMapType( + CalcitePlanContext context, RexNode origin, SqlTypeName targetType) { + MapSqlType originalMapType = (MapSqlType) origin.getType(); + ArraySqlType newValueType = + new ArraySqlType(context.rexBuilder.getTypeFactory().createSqlType(targetType), true); + MapSqlType newMapType = new MapSqlType(originalMapType.getKeyType(), newValueType, true); + return new RexInputRef(((RexInputRef) origin).getIndex(), newMapType); + } + private void flattenParsedPattern( String originalPatternResultAlias, RexNode parsedNode, @@ -2481,7 +3048,7 @@ private void flattenParsedPattern( PPLFuncImpTable.INSTANCE.resolve( context.rexBuilder, BuiltinFunctionName.INTERNAL_ITEM, - parsedNode, + explicitMapType(context, parsedNode, SqlTypeName.VARCHAR), context.rexBuilder.makeLiteral(PatternUtils.SAMPLE_LOGS)), true, true); diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index c5b89e08f73..ef6def9d4dd 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -7,7 +7,6 @@ import static java.util.Objects.requireNonNull; import static org.apache.calcite.sql.SqlKind.AS; -import static org.apache.commons.lang3.StringUtils.substringAfterLast; import static org.opensearch.sql.ast.expression.SpanUnit.NONE; import static org.opensearch.sql.ast.expression.SpanUnit.UNKNOWN; import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.TYPE_FACTORY; @@ -34,6 +33,7 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.ArraySqlType; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.util.DateString; import org.apache.calcite.util.TimeString; import org.apache.calcite.util.TimestampString; @@ -68,12 +68,17 @@ import org.opensearch.sql.ast.expression.subquery.ExistsSubquery; import org.opensearch.sql.ast.expression.subquery.InSubquery; import org.opensearch.sql.ast.expression.subquery.ScalarSubquery; +import org.opensearch.sql.ast.expression.subquery.SubqueryExpression; import org.opensearch.sql.ast.tree.UnresolvedPlan; +import org.opensearch.sql.calcite.plan.LogicalSystemLimit; +import org.opensearch.sql.calcite.plan.LogicalSystemLimit.SystemLimitType; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.calcite.utils.PlanUtils; +import org.opensearch.sql.calcite.utils.SubsearchUtils; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.CalciteUnsupportedException; +import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.PPLFuncImpTable; @@ -249,102 +254,7 @@ public RexNode visitEqualTo(EqualTo node, CalcitePlanContext context) { /** Resolve qualified name. Note, the name should be case-sensitive. */ @Override public RexNode visitQualifiedName(QualifiedName node, CalcitePlanContext context) { - // 1. resolve QualifiedName in join condition - if (context.isResolvingJoinCondition()) { - List parts = node.getParts(); - if (parts.size() == 1) { - // 1.1 Handle the case of `id = cid` - try { - return context.relBuilder.field(2, 0, parts.getFirst()); - } catch (IllegalArgumentException ee) { - return context.relBuilder.field(2, 1, parts.getFirst()); - } - } else if (parts.size() == 2) { - // 1.2 Handle the case of `t1.id = t2.id` or `alias1.id = alias2.id` - try { - return context.relBuilder.field(2, parts.get(0), parts.get(1)); - } catch (IllegalArgumentException e) { - // Similar to the step 2.3. - List candidates = - context.relBuilder.peek(1).getRowType().getFieldNames().stream() - .filter(col -> substringAfterLast(col, ".").equals(parts.getLast())) - .toList(); - for (String candidate : candidates) { - try { - // field("nation2", "n2.n_name"); // pass - return context.relBuilder.field(2, parts.get(0), candidate); - } catch (IllegalArgumentException e1) { - // field("nation2", "n_name"); // do nothing when fail (n_name is field of nation1) - } - } - throw new UnsupportedOperationException("Unsupported qualified name: " + node); - } - } else if (parts.size() == 3) { - throw new UnsupportedOperationException("Unsupported qualified name: " + node); - } - } - - // TODO: Need to support nested fields https://github.com/opensearch-project/sql/issues/3459 - // 2. resolve QualifiedName in non-join condition - String qualifiedName = node.toString(); - if (context.getRexLambdaRefMap().containsKey(qualifiedName)) { - return context.getRexLambdaRefMap().get(qualifiedName); - } - List currentFields = context.relBuilder.peek().getRowType().getFieldNames(); - - if (!currentFields.contains(qualifiedName) && context.isInCoalesceFunction()) { - return context.rexBuilder.makeNullLiteral( - context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR)); - } - - if (currentFields.contains(qualifiedName)) { - // 2.1 resolve QualifiedName from stack top - // Note: QualifiedName with multiple parts also could be applied in step 2.1, - // for example `n2.n_name` or `nation2.n_name` in the output of join can be resolved here. - return context.relBuilder.field(qualifiedName); - } else if (node.getParts().size() == 2) { - // 2.2 resolve QualifiedName with an alias or table name - List parts = node.getParts(); - try { - return context.relBuilder.field(1, parts.get(0), parts.get(1)); - } catch (IllegalArgumentException e) { - // 2.3 For field which renamed with , to resolve the field with table - // identifier - // `nation2.n_name`, - // we convert it to resolve , e.g. `nation2.n2.n_name` - // `n2.n_name` was the renamed field name from the duplicated field `(nation2.)n_name0` of - // join output. - // Build the candidates which contains `n_name`: e.g. `(nation1.)n_name`, `n2.n_name` - List candidates = - context.relBuilder.peek().getRowType().getFieldNames().stream() - .filter(col -> substringAfterLast(col, ".").equals(parts.getLast())) - .toList(); - for (String candidate : candidates) { - try { - // field("nation2", "n2.n_name"); // pass - return context.relBuilder.field(parts.get(0), candidate); - } catch (IllegalArgumentException e1) { - // field("nation2", "n_name"); // do nothing when fail (n_name is field of nation1) - } - } - // 2.4 resolve QualifiedName with outer alias - // check existing of parts.get(0) - return context - .peekCorrelVar() - .map(correlVar -> context.relBuilder.field(correlVar, parts.get(1))) - .orElseThrow(() -> e); // Re-throw the exception if no correlated variable exists - } - } else if (currentFields.stream().noneMatch(f -> f.startsWith(qualifiedName))) { - // 2.5 try resolving combination of 2.1 and 2.4 to resolve rest cases - return context - .peekCorrelVar() - .map(correlVar -> context.relBuilder.field(correlVar, qualifiedName)) - .orElseGet(() -> context.relBuilder.field(qualifiedName)); - } else { - throw new IllegalArgumentException( - String.format( - "field [%s] not found; input fields are: %s", qualifiedName, currentFields)); - } + return QualifiedNameResolver.resolve(node, context); } @Override @@ -535,9 +445,26 @@ public RexNode visitWindowFunction(WindowFunction node, CalcitePlanContext conte (arguments.isEmpty() || arguments.size() == 1) ? Collections.emptyList() : arguments.subList(1, arguments.size()); - PPLFuncImpTable.INSTANCE.validateAggFunctionSignature(functionName, field, args); - return PlanUtils.makeOver( - context, functionName, field, args, partitions, List.of(), node.getWindowFrame()); + List nodes = + PPLFuncImpTable.INSTANCE.validateAggFunctionSignature( + functionName, field, args, context.rexBuilder); + return nodes != null + ? PlanUtils.makeOver( + context, + functionName, + nodes.getFirst(), + nodes.size() <= 1 ? Collections.emptyList() : nodes.subList(1, nodes.size()), + partitions, + List.of(), + node.getWindowFrame()) + : PlanUtils.makeOver( + context, + functionName, + field, + args, + partitions, + List.of(), + node.getWindowFrame()); }) .orElseThrow( () -> @@ -559,7 +486,7 @@ private RexNode extractRexNodeFromAlias(RexNode node) { public RexNode visitInSubquery(InSubquery node, CalcitePlanContext context) { List nodes = node.getChild().stream().map(child -> analyze(child, context)).toList(); UnresolvedPlan subquery = node.getQuery(); - RelNode subqueryRel = resolveSubqueryPlan(subquery, context); + RelNode subqueryRel = resolveSubqueryPlan(subquery, node, context); if (subqueryRel.getRowType().getFieldCount() != nodes.size()) { throw new SemanticCheckException( "The number of columns in the left hand side of an IN subquery does not match the number" @@ -583,7 +510,7 @@ public RexNode visitScalarSubquery(ScalarSubquery node, CalcitePlanContext conte return context.relBuilder.scalarQuery( b -> { UnresolvedPlan subquery = node.getQuery(); - return resolveSubqueryPlan(subquery, context); + return resolveSubqueryPlan(subquery, node, context); }); } @@ -592,11 +519,12 @@ public RexNode visitExistsSubquery(ExistsSubquery node, CalcitePlanContext conte return context.relBuilder.exists( b -> { UnresolvedPlan subquery = node.getQuery(); - return resolveSubqueryPlan(subquery, context); + return resolveSubqueryPlan(subquery, node, context); }); } - private RelNode resolveSubqueryPlan(UnresolvedPlan subquery, CalcitePlanContext context) { + private RelNode resolveSubqueryPlan( + UnresolvedPlan subquery, SubqueryExpression subqueryExpression, CalcitePlanContext context) { boolean isNestedSubquery = context.isResolvingSubquery(); context.setResolvingSubquery(true); // clear and store the outer state @@ -604,9 +532,26 @@ private RelNode resolveSubqueryPlan(UnresolvedPlan subquery, CalcitePlanContext if (isResolvingJoinConditionOuter) { context.setResolvingJoinCondition(false); } - RelNode subqueryRel = subquery.accept(planVisitor, context); + subquery.accept(planVisitor, context); + // add subsearch.maxout limit to exists-in subsearch, 0 and negative means unlimited + if (context.sysLimit.subsearchLimit() > 0 && !(subqueryExpression instanceof ScalarSubquery)) { + // Cannot add system limit to the top of subquery simply. + // Instead, add system limit under the correlated conditions. + SubsearchUtils.SystemLimitInsertionShuttle shuttle = + new SubsearchUtils.SystemLimitInsertionShuttle(context); + RelNode replacement = context.relBuilder.peek().accept(shuttle); + if (!shuttle.isCorrelatedConditionFound()) { + // If no correlated condition found, add system limit to the top of subquery. + replacement = + LogicalSystemLimit.create( + SystemLimitType.SUBSEARCH_MAXOUT, + replacement, + context.relBuilder.literal(context.sysLimit.subsearchLimit())); + } + PlanUtils.replaceTop(context.relBuilder, replacement); + } // pop the inner plan - context.relBuilder.build(); + RelNode subqueryRel = context.relBuilder.build(); // clear the exists subquery resolving state // restore to the previous state if (isResolvingJoinConditionOuter) { @@ -634,7 +579,13 @@ public RexNode visitCast(Cast node, CalcitePlanContext context) { public RexNode visitCase(Case node, CalcitePlanContext context) { List caseOperands = new ArrayList<>(); for (When when : node.getWhenClauses()) { - caseOperands.add(analyze(when.getCondition(), context)); + RexNode condition = analyze(when.getCondition(), context); + if (!SqlTypeUtil.isBoolean(condition.getType())) { + throw new ExpressionEvaluationException( + StringUtils.format( + "Condition expected a boolean type, but got %s", condition.getType())); + } + caseOperands.add(condition); caseOperands.add(analyze(when.getResult(), context)); } RexNode elseExpr = diff --git a/core/src/main/java/org/opensearch/sql/calcite/ExtendedRexBuilder.java b/core/src/main/java/org/opensearch/sql/calcite/ExtendedRexBuilder.java index d8bea5e3371..c353271d370 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/ExtendedRexBuilder.java +++ b/core/src/main/java/org/opensearch/sql/calcite/ExtendedRexBuilder.java @@ -54,38 +54,59 @@ public RelDataType commonType(RexNode... nodes) { public SqlIntervalQualifier createIntervalUntil(SpanUnit unit) { TimeUnit timeUnit; switch (unit) { + case MICROSECOND: + case US: + timeUnit = TimeUnit.MICROSECOND; + break; case MILLISECOND: case MS: timeUnit = TimeUnit.MILLISECOND; break; + case SECONDS: case SECOND: + case SECS: + case SEC: case S: timeUnit = TimeUnit.SECOND; break; + case MINUTES: case MINUTE: + case MINS: + case MIN: case m: timeUnit = TimeUnit.MINUTE; break; + case HOURS: case HOUR: + case HRS: + case HR: case H: timeUnit = TimeUnit.HOUR; break; + case DAYS: case DAY: case D: timeUnit = TimeUnit.DAY; break; + case WEEKS: case WEEK: case W: timeUnit = TimeUnit.WEEK; break; + case MONTHS: case MONTH: + case MON: case M: timeUnit = TimeUnit.MONTH; break; + case QUARTERS: case QUARTER: + case QTRS: + case QTR: case Q: timeUnit = TimeUnit.QUARTER; break; + case YEARS: case YEAR: case Y: timeUnit = TimeUnit.YEAR; @@ -126,6 +147,9 @@ public RexNode makeCast( // ImmutableList.of(exp, makeZeroLiteral(sourceType))); } } else if (OpenSearchTypeFactory.isUserDefinedType(type)) { + if (RexLiteral.isNullLiteral(exp)) { + return super.makeCast(pos, type, exp, matchNullability, safe, format); + } var udt = ((AbstractExprRelDataType) type).getUdt(); var argExprType = OpenSearchTypeFactory.convertRelDataTypeToExprType(sourceType); return switch (udt) { diff --git a/core/src/main/java/org/opensearch/sql/calcite/QualifiedNameResolver.java b/core/src/main/java/org/opensearch/sql/calcite/QualifiedNameResolver.java new file mode 100644 index 00000000000..39dff19fff6 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/QualifiedNameResolver.java @@ -0,0 +1,310 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.sql.ast.expression.QualifiedName; +import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.PPLFuncImpTable; + +/** + * Utility class for resolving qualified names in Calcite query planning. Extracts the qualified + * name resolution logic from CalciteRexNodeVisitor to provide a centralized and reusable + * implementation. + */ +public class QualifiedNameResolver { + + private static final Logger log = LogManager.getLogger(QualifiedNameResolver.class); + + /** + * Resolves a qualified name to a RexNode based on the current context. + * + * @param nameNode The QualifiedName to resolve + * @param context The CalcitePlanContext containing the current state + * @return RexNode representing the resolved qualified name + * @throws IllegalArgumentException if the field is not found in the current context + */ + public static RexNode resolve(QualifiedName nameNode, CalcitePlanContext context) { + log.debug( + "QualifiedNameResolver.resolve() called with nameNode={}, isResolvingJoinCondition={}", + nameNode, + context.isResolvingJoinCondition()); + + if (context.isResolvingJoinCondition()) { + return resolveInJoinCondition(nameNode, context); + } else { + return resolveInNonJoinCondition(nameNode, context); + } + } + + /** Resolves qualified name in join condition context. */ + private static RexNode resolveInJoinCondition( + QualifiedName nameNode, CalcitePlanContext context) { + log.debug("resolveInJoinCondition() called with nameNode={}", nameNode); + + return resolveFieldWithAlias(nameNode, context, 2) + .or(() -> resolveFieldWithoutAlias(nameNode, context, 2)) + .orElseThrow(() -> getNotFoundException(nameNode)); + } + + /** Resolves qualified name in non-join condition context. */ + private static RexNode resolveInNonJoinCondition( + QualifiedName nameNode, CalcitePlanContext context) { + log.debug("resolveInNonJoinCondition() called with nameNode={}", nameNode); + + return resolveLambdaVariable(nameNode, context) + .or(() -> resolveFieldDirectly(nameNode, context, 1)) + .or(() -> resolveFieldWithAlias(nameNode, context, 1)) + .or(() -> resolveFieldWithoutAlias(nameNode, context, 1)) + .or(() -> resolveRenamedField(nameNode, context)) + .or(() -> resolveCorrelationField(nameNode, context)) + .or(() -> replaceWithNullLiteralInCoalesce(context)) + .orElseThrow(() -> getNotFoundException(nameNode)); + } + + private static String joinParts(List parts, int start, int length) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < length; i++) { + if (start < i) { + sb.append("."); + } + sb.append(parts.get(start + i)); + } + return sb.toString(); + } + + private static String joinParts(List parts, int start) { + return joinParts(parts, start, parts.size() - start); + } + + private static Optional resolveFieldDirectly( + QualifiedName nameNode, CalcitePlanContext context, int inputCount) { + List parts = nameNode.getParts(); + log.debug( + "resolveFieldDirectly() called with nameNode={}, parts={}, inputCount={}", + nameNode, + parts, + inputCount); + + List currentFields = context.relBuilder.peek().getRowType().getFieldNames(); + if (currentFields.contains(nameNode.toString())) { + try { + return Optional.of(context.relBuilder.field(nameNode.toString())); + } catch (IllegalArgumentException e) { + log.debug("resolveFieldDirectly() failed: {}", e.getMessage()); + } + } + return Optional.empty(); + } + + private static Optional resolveFieldWithAlias( + QualifiedName nameNode, CalcitePlanContext context, int inputCount) { + List parts = nameNode.getParts(); + log.debug( + "resolveFieldWithAlias() called with nameNode={}, parts={}, inputCount={}", + nameNode, + parts, + inputCount); + + if (parts.size() >= 2) { + // Consider first part as table alias + String alias = parts.get(0); + log.debug("resolveFieldWithAlias() trying alias={}", alias); + + // Try to resolve the longest match first + for (int length = parts.size() - 1; 1 <= length; length--) { + String field = joinParts(parts, 1, length); + log.debug("resolveFieldWithAlias() trying field={} with length={}", field, length); + + Optional fieldNode = tryToResolveField(alias, field, context, inputCount); + if (fieldNode.isPresent()) { + return Optional.of(resolveFieldAccess(context, parts, 1, length, fieldNode.get())); + } + } + } + return Optional.empty(); + } + + private static Optional tryToResolveField( + String alias, String fieldName, CalcitePlanContext context, int inputCount) { + log.debug( + "tryToResolveField() called with alias={}, fieldName={}, inputCount={}", + alias, + fieldName, + inputCount); + try { + return Optional.of(context.relBuilder.field(inputCount, alias, fieldName)); + } catch (IllegalArgumentException e) { + log.debug("tryToResolveField() failed: {}", e.getMessage()); + } + return Optional.empty(); + } + + private static Optional resolveFieldWithoutAlias( + QualifiedName nameNode, CalcitePlanContext context, int inputCount) { + log.debug( + "resolveFieldWithoutAlias() called with nameNode={}, inputCount={}", nameNode, inputCount); + + List> inputFieldNames = collectInputFieldNames(context, inputCount); + + List parts = nameNode.getParts(); + for (int length = parts.size(); 1 <= length; length--) { + String fieldName = joinParts(parts, 0, length); + log.debug("resolveFieldWithoutAlias() trying fieldName={} with length={}", fieldName, length); + + int foundInput = findInputContainingFieldName(inputCount, inputFieldNames, fieldName); + log.debug("resolveFieldWithoutAlias() foundInput={}", foundInput); + if (foundInput != -1) { + RexNode fieldNode = context.relBuilder.field(inputCount, foundInput, fieldName); + return Optional.of(resolveFieldAccess(context, parts, 0, length, fieldNode)); + } + } + return Optional.empty(); + } + + private static int findInputContainingFieldName( + int inputCount, List> inputFieldNames, String fieldName) { + int foundInput = -1; + for (int i = 0; i < inputCount; i++) { + if (inputFieldNames.get(i).contains(fieldName)) { + if (foundInput != -1) { + throw new IllegalArgumentException("Ambiguous field: " + fieldName); + } else { + foundInput = i; + } + } + } + return foundInput; + } + + private static List> collectInputFieldNames( + CalcitePlanContext context, int inputCount) { + List> inputFieldNames = new ArrayList<>(); + for (int i = 0; i < inputCount; i++) { + int inputOrdinal = inputCount - i - 1; + Set fieldNames = + context.relBuilder.peek(inputOrdinal).getRowType().getFieldList().stream() + .map(RelDataTypeField::getName) + .collect(Collectors.toSet()); + inputFieldNames.add(fieldNames); + log.debug("collectInputFieldNames() input[{}] fieldNames={}", inputOrdinal, fieldNames); + } + return inputFieldNames; + } + + /** Try to resolve renamed field due to duplicate field name while join. e.g. alias.fieldName */ + private static Optional resolveRenamedField( + QualifiedName nameNode, CalcitePlanContext context) { + log.debug("resolveRenamedField() called with nameNode={}", nameNode); + + List parts = nameNode.getParts(); + if (parts.size() >= 2) { + List candidates = findCandidatesByRenamedFieldName(nameNode, context); + String alias = parts.get(0); + for (String candidate : candidates) { + try { + return Optional.of(context.relBuilder.field(alias, candidate)); + } catch (IllegalArgumentException e1) { + // Indicates the field was not found. + } + } + } + return Optional.empty(); + } + + /** + * Find the original name before fieldName is renamed due to duplicate field name. Example: + * renamedFieldname = "alias.fieldName", originalFieldName = "fieldName" + */ + private static List findCandidatesByRenamedFieldName( + QualifiedName renamedFieldName, CalcitePlanContext context) { + String originalFieldName = joinParts(renamedFieldName.getParts(), 1); + return context.relBuilder.peek().getRowType().getFieldNames().stream() + .filter(col -> getNameBeforeRename(col).equals(originalFieldName)) + .toList(); + } + + private static String getNameBeforeRename(String fieldName) { + return fieldName.substring(fieldName.indexOf(".") + 1); + } + + private static Optional resolveCorrelationField( + QualifiedName nameNode, CalcitePlanContext context) { + log.debug("resolveCorrelationField() called with nameNode={}", nameNode); + List parts = nameNode.getParts(); + return context + .peekCorrelVar() + .map( + correlation -> { + List fieldNameList = correlation.getType().getFieldNames(); + // Try full match, then consider first part as table alias + for (int start = 0; start <= 1; start++) { + // Try to resolve the longest match first + for (int length = parts.size() - start; 1 <= length; length--) { + String fieldName = joinParts(parts, start, length); + log.debug("resolveCorrelationField() trying fieldName={}", fieldName); + if (fieldNameList.contains(fieldName)) { + RexNode field = context.relBuilder.field(correlation, fieldName); + return resolveFieldAccess(context, parts, start, length, field); + } + } + } + return null; + }); + } + + private static RexNode resolveFieldAccess( + CalcitePlanContext context, List parts, int start, int length, RexNode field) { + if (length == parts.size() - start) { + return field; + } else { + String itemName = joinParts(parts, length + start, parts.size() - length); + return context.relBuilder.alias( + createItemAccess(field, itemName, context), + String.join(QualifiedName.DELIMITER, parts.subList(start, parts.size()))); + } + } + + private static RexNode createItemAccess( + RexNode field, String itemName, CalcitePlanContext context) { + log.debug("createItemAccess() called with itemName={}", itemName); + return PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, + BuiltinFunctionName.INTERNAL_ITEM, + field, + context.rexBuilder.makeLiteral(itemName)); + } + + private static Optional resolveLambdaVariable( + QualifiedName nameNode, CalcitePlanContext context) { + log.debug("resolveLambdaVariable() called with nameNode={}", nameNode); + String qualifiedName = nameNode.toString(); + return Optional.ofNullable(context.getRexLambdaRefMap().get(qualifiedName)); + } + + private static Optional replaceWithNullLiteralInCoalesce(CalcitePlanContext context) { + log.debug("replaceWithNullLiteralInCoalesce() called"); + if (context.isInCoalesceFunction()) { + return Optional.of( + context.rexBuilder.makeNullLiteral( + context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR))); + } + return Optional.empty(); + } + + private static RuntimeException getNotFoundException(QualifiedName node) { + return new IllegalArgumentException(String.format("Field [%s] not found.", node.toString())); + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/SchemaUnifier.java b/core/src/main/java/org/opensearch/sql/calcite/SchemaUnifier.java new file mode 100644 index 00000000000..05380ce8c48 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/SchemaUnifier.java @@ -0,0 +1,150 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexNode; + +/** + * Utility class for unifying schemas across multiple RelNodes. Throws an exception when type + * conflicts are detected. + */ +public class SchemaUnifier { + + /** + * Builds a unified schema for multiple nodes. Throws an exception if type conflicts are detected. + * + * @param nodes List of RelNodes to unify schemas for + * @param context Calcite plan context + * @return List of projected RelNodes with unified schema + * @throws IllegalArgumentException if type conflicts are detected + */ + public static List buildUnifiedSchemaWithConflictResolution( + List nodes, CalcitePlanContext context) { + if (nodes.isEmpty()) { + return new ArrayList<>(); + } + + if (nodes.size() == 1) { + return nodes; + } + + // Step 1: Build the unified schema by processing all nodes (throws on conflict) + List unifiedSchema = buildUnifiedSchema(nodes); + + // Step 2: Create projections for each node to align with unified schema + List projectedNodes = new ArrayList<>(); + List fieldNames = + unifiedSchema.stream().map(SchemaField::getName).collect(Collectors.toList()); + + for (RelNode node : nodes) { + List projection = buildProjectionForNode(node, unifiedSchema, context); + RelNode projectedNode = context.relBuilder.push(node).project(projection, fieldNames).build(); + projectedNodes.add(projectedNode); + } + + return projectedNodes; + } + + /** + * Builds a unified schema by merging fields from all nodes. Throws an exception if fields with + * the same name have different types. + * + * @param nodes List of RelNodes to merge schemas from + * @return List of SchemaField representing the unified schema + * @throws IllegalArgumentException if type conflicts are detected + */ + private static List buildUnifiedSchema(List nodes) { + List schema = new ArrayList<>(); + Map seenFields = new HashMap<>(); + + for (RelNode node : nodes) { + for (RelDataTypeField field : node.getRowType().getFieldList()) { + String fieldName = field.getName(); + RelDataType fieldType = field.getType(); + + RelDataType existingType = seenFields.get(fieldName); + if (existingType == null) { + // New field - add to schema + schema.add(new SchemaField(fieldName, fieldType)); + seenFields.put(fieldName, fieldType); + } else if (!areTypesCompatible(existingType, fieldType)) { + // Same field name but different type - throw exception + throw new IllegalArgumentException( + String.format( + "Unable to process column '%s' due to incompatible types: '%s' and '%s'", + fieldName, existingType.getSqlTypeName(), fieldType.getSqlTypeName())); + } + // If we've seen this exact (name, type) combination, skip it + } + } + + return schema; + } + + private static boolean areTypesCompatible(RelDataType type1, RelDataType type2) { + return type1.getSqlTypeName() != null && type1.getSqlTypeName().equals(type2.getSqlTypeName()); + } + + /** + * Builds a projection for a node to align with the unified schema. For each field in the unified + * schema: - If the node has a matching field with the same type, use it - Otherwise, project NULL + * + * @param node The node to build projection for + * @param unifiedSchema List of SchemaField representing the unified schema + * @param context Calcite plan context + * @return List of RexNode representing the projection + */ + private static List buildProjectionForNode( + RelNode node, List unifiedSchema, CalcitePlanContext context) { + Map nodeFieldMap = + node.getRowType().getFieldList().stream() + .collect(Collectors.toMap(RelDataTypeField::getName, field -> field)); + + List projection = new ArrayList<>(); + for (SchemaField schemaField : unifiedSchema) { + String fieldName = schemaField.getName(); + RelDataType expectedType = schemaField.getType(); + RelDataTypeField nodeField = nodeFieldMap.get(fieldName); + + if (nodeField != null && areTypesCompatible(nodeField.getType(), expectedType)) { + // Field exists with compatible type - use it + projection.add(context.rexBuilder.makeInputRef(node, nodeField.getIndex())); + } else { + // Field missing or type mismatch - project NULL + projection.add(context.rexBuilder.makeNullLiteral(expectedType)); + } + } + + return projection; + } + + /** Represents a field in the unified schema with name and type. */ + private static class SchemaField { + private final String name; + private final RelDataType type; + + SchemaField(String name, RelDataType type) { + this.name = name; + this.type = type; + } + + String getName() { + return name; + } + + RelDataType getType() { + return type; + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/SysLimit.java b/core/src/main/java/org/opensearch/sql/calcite/SysLimit.java new file mode 100644 index 00000000000..4f2cd951079 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/SysLimit.java @@ -0,0 +1,26 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite; + +import org.opensearch.sql.common.setting.Settings; + +public record SysLimit(Integer querySizeLimit, Integer subsearchLimit, Integer joinSubsearchLimit) { + /** Create SysLimit from Settings. */ + public static SysLimit fromSettings(Settings settings) { + return settings == null + ? UNLIMITED_SUBSEARCH + : new SysLimit( + settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT), + settings.getSettingValue(Settings.Key.PPL_SUBSEARCH_MAXOUT), + settings.getSettingValue(Settings.Key.PPL_JOIN_SUBSEARCH_MAXOUT)); + } + + /** No limitation on subsearch */ + public static SysLimit UNLIMITED_SUBSEARCH = new SysLimit(10000, 0, 0); + + /** For testing only */ + public static SysLimit DEFAULT = new SysLimit(10000, 10000, 50000); +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/plan/LogicalSystemLimit.java b/core/src/main/java/org/opensearch/sql/calcite/plan/LogicalSystemLimit.java index f910d6dcc61..c33854ebe52 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/plan/LogicalSystemLimit.java +++ b/core/src/main/java/org/opensearch/sql/calcite/plan/LogicalSystemLimit.java @@ -30,7 +30,11 @@ public enum SystemLimitType { * *

This type is used to indicate that the limit is applied to the system level. */ - QUERY_SIZE_LIMIT + QUERY_SIZE_LIMIT, + /** The max output from subsearch to join against. */ + JOIN_SUBSEARCH_MAXOUT, + /** Max output to return from a subsearch. */ + SUBSEARCH_MAXOUT, } @Getter private final SystemLimitType type; diff --git a/core/src/main/java/org/opensearch/sql/calcite/plan/OpenSearchRules.java b/core/src/main/java/org/opensearch/sql/calcite/plan/OpenSearchRules.java index 3f64e23493d..8d41b30ab91 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/plan/OpenSearchRules.java +++ b/core/src/main/java/org/opensearch/sql/calcite/plan/OpenSearchRules.java @@ -10,11 +10,13 @@ import org.apache.calcite.plan.RelOptRule; public class OpenSearchRules { - private static final PPLAggregateConvertRule AGGREGATE_CONVERT_RULE = + public static final PPLAggregateConvertRule AGGREGATE_CONVERT_RULE = PPLAggregateConvertRule.Config.SUM_CONVERTER.toRule(); + public static final PPLAggGroupMergeRule AGG_GROUP_MERGE_RULE = + PPLAggGroupMergeRule.Config.GROUP_MERGE.toRule(); public static final List OPEN_SEARCH_OPT_RULES = - ImmutableList.of(AGGREGATE_CONVERT_RULE); + ImmutableList.of(AGGREGATE_CONVERT_RULE, AGG_GROUP_MERGE_RULE); // prevent instantiation private OpenSearchRules() {} diff --git a/core/src/main/java/org/opensearch/sql/calcite/plan/PPLAggGroupMergeRule.java b/core/src/main/java/org/opensearch/sql/calcite/plan/PPLAggGroupMergeRule.java new file mode 100644 index 00000000000..f1671e0eb63 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/plan/PPLAggGroupMergeRule.java @@ -0,0 +1,156 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.plan; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.mapping.Mapping; +import org.apache.calcite.util.mapping.Mappings; +import org.apache.commons.lang3.tuple.Pair; +import org.immutables.value.Value; +import org.opensearch.sql.calcite.utils.CalciteUtils; + +/** + * Planner rule that merge multiple agg group fields into a single one, on which all other group + * fields depend. e.g. + * + *

stats ... by a, f1(a), f2(a) -> stats ... by a | eval `f1(a)` = f1(a), `f2(a)` = f2(a) + * + *

TODO: this rule could be expanded further for more cases: 1. support multiple base group + * fields, e.g. stats ... by a, f1(a), b, f2(b), f3(a, b) -> stats ... by a, b | eval `f1(a)` = + * f1(a), `f2(b)` = f2(b), `f3(a, b)` = f3(a, b) 2. support no base fields, e.g. stats ... by f1(a), + * f2(a) -> stats ... by a | eval `f1(a)` = f1(a), `f2(a)` = f2(a) | fields - a Note that one of + * these UDFs' output must have equivalent cardinality as `a`. + */ +@Value.Enclosing +public class PPLAggGroupMergeRule extends RelRule { + + /** Creates a OpenSearchAggregateConvertRule. */ + protected PPLAggGroupMergeRule(Config config) { + super(config); + } + + @Override + public void onMatch(RelOptRuleCall call) { + if (call.rels.length == 2) { + final LogicalAggregate aggregate = call.rel(0); + final LogicalProject project = call.rel(1); + apply(call, aggregate, project); + } else { + throw new AssertionError( + String.format( + "The length of rels should be %s but got %s", + this.operands.size(), call.rels.length)); + } + } + + public void apply(RelOptRuleCall call, LogicalAggregate aggregate, LogicalProject project) { + List groupSet = aggregate.getGroupSet().asList(); + List groupNodes = + groupSet.stream().map(group -> project.getProjects().get(group)).toList(); + Pair, List> baseFieldsAndOthers = + CalciteUtils.partition( + groupSet, i -> project.getProjects().get(i).getKind() == SqlKind.INPUT_REF); + List baseGroupList = baseFieldsAndOthers.getLeft(); + // TODO: support more base fields in the future. + if (baseGroupList.size() != 1) return; + Integer baseGroupField = baseGroupList.get(0); + RexInputRef baseGroupRef = (RexInputRef) project.getProjects().get(baseGroupField); + List otherGroupList = baseFieldsAndOthers.getRight(); + boolean allDependOnBaseField = + otherGroupList.stream() + .map(i -> project.getProjects().get(i)) + .allMatch(node -> isDependentField(node, List.of(baseGroupRef))); + if (!allDependOnBaseField) return; + + final RelBuilder relBuilder = call.builder(); + relBuilder.push(project); + + relBuilder.aggregate( + relBuilder.groupKey(ImmutableBitSet.of(baseGroupField)), aggregate.getAggCallList()); + + /* Build the final project-aggregate-project */ + final Mapping mapping = + Mappings.target( + List.of(baseGroupRef.getIndex()), + baseGroupRef.getIndex() + 1); // set source count greater than the max ref index + List parentProjections = new ArrayList<>(RexUtil.apply(mapping, groupNodes)); + List aggCallRefs = + relBuilder.fields( + IntStream.range(baseGroupList.size(), relBuilder.peek().getRowType().getFieldCount()) + .boxed() + .toList()); + parentProjections.addAll(aggCallRefs); + relBuilder.project(parentProjections); + call.transformTo(relBuilder.build()); + } + + /** Rule configuration. */ + @Value.Immutable + public interface Config extends RelRule.Config { + Config GROUP_MERGE = + ImmutablePPLAggGroupMergeRule.Config.builder() + .build() + .withOperandSupplier( + b0 -> + b0.operand(LogicalAggregate.class) + .predicate(Config::containsMultipleGroupSets) + .oneInput( + b1 -> + b1.operand(LogicalProject.class) + .predicate(Config::containsDependentFields) + .anyInputs())); + + static boolean containsMultipleGroupSets(LogicalAggregate aggregate) { + return aggregate.getGroupSet().cardinality() > 1; + } + + // Only rough predication here since we don't know which fields are group fields currently. + static boolean containsDependentFields(LogicalProject project) { + Set baseFields = + project.getProjects().stream() + .filter(node -> node.getKind() == SqlKind.INPUT_REF) + .collect(Collectors.toUnmodifiableSet()); + return project.getProjects().stream() + .anyMatch(node -> PPLAggGroupMergeRule.isDependentField(node, baseFields)); + } + + @Override + default PPLAggGroupMergeRule toRule() { + return new PPLAggGroupMergeRule(this); + } + } + + public static boolean isDependentField(RexNode node, Collection baseFields) { + // Always view literal field as dependent field here since we can always implement a function + // to transform a field into such a literal + if (node.getKind() == SqlKind.LITERAL) return true; + if (node.getKind() == SqlKind.INPUT_REF && baseFields.contains(node)) return true; + // Use !isAggregator to rule out window functions like row_number() + if (node instanceof RexCall + && ((RexCall) node).getOperator().isDeterministic() + && !((RexCall) node).getOperator().isAggregator()) { + return ((RexCall) node) + .getOperands().stream().allMatch(op -> isDependentField(op, baseFields)); + } + return false; + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/plan/PPLAggregateConvertRule.java b/core/src/main/java/org/opensearch/sql/calcite/plan/PPLAggregateConvertRule.java index f15b040f98c..2f385054482 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/plan/PPLAggregateConvertRule.java +++ b/core/src/main/java/org/opensearch/sql/calcite/plan/PPLAggregateConvertRule.java @@ -7,14 +7,10 @@ import com.google.common.collect.ImmutableList; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Set; import java.util.function.Function; import java.util.stream.IntStream; import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.RelRule; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.AggregateCall; @@ -30,8 +26,6 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.tools.RelBuilder; -import org.apache.calcite.util.ImmutableBitSet; -import org.apache.calcite.util.mapping.Mappings; import org.apache.commons.lang3.tuple.Pair; import org.immutables.value.Value; @@ -188,36 +182,9 @@ public void apply(RelOptRuleCall call, LogicalAggregate aggregate, LogicalProjec } } - /* Eliminate unused fields in the child project */ - ImmutableBitSet newGroupSet = aggregate.getGroupSet(); - ; - ImmutableList newGroupSets = aggregate.getGroupSets(); - ; - final Set fieldsUsed = - RelOptUtil.getAllFields2(aggregate.getGroupSet(), distinctAggregateCalls); - if (fieldsUsed.size() < newChildProjects.size()) { - // Some fields are computed but not used. Prune them. - final Map sourceFieldToTargetFieldMap = new HashMap<>(); - for (int source : fieldsUsed) { - sourceFieldToTargetFieldMap.put(source, sourceFieldToTargetFieldMap.size()); - } - newGroupSet = aggregate.getGroupSet().permute(sourceFieldToTargetFieldMap); - newGroupSets = - ImmutableBitSet.ORDERING.immutableSortedCopy( - ImmutableBitSet.permute(aggregate.getGroupSets(), sourceFieldToTargetFieldMap)); - final Mappings.TargetMapping targetMapping = - Mappings.target(sourceFieldToTargetFieldMap, newChildProjects.size(), fieldsUsed.size()); - final List oldAggregateCalls = new ArrayList<>(distinctAggregateCalls); - distinctAggregateCalls.clear(); - for (AggregateCall aggregateCall : oldAggregateCalls) { - distinctAggregateCalls.add(aggregateCall.transform(targetMapping)); - } - // Project the used fields - relBuilder.project(relBuilder.fields(fieldsUsed.stream().toList())); - } + relBuilder.aggregate(relBuilder.groupKey(aggregate.getGroupSet()), distinctAggregateCalls); - /* Build the final project-aggregate-project after eliminating unused fields */ - relBuilder.aggregate(relBuilder.groupKey(newGroupSet, newGroupSets), distinctAggregateCalls); + /* Build the final project-aggregate-project */ List parentProjects = new ArrayList<>(relBuilder.fields(IntStream.range(0, groupSetOffset).boxed().toList())); parentProjects.addAll( diff --git a/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/PercentileApproxFunction.java b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/PercentileApproxFunction.java index 9fadd083362..613291dcce1 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/PercentileApproxFunction.java +++ b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/PercentileApproxFunction.java @@ -62,7 +62,7 @@ public Object result(PencentileApproAccumulator acc) { float floatRet = (float) retValue; return floatRet; default: - return acc.value(); + return retValue; } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java b/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java index 6513f51a5a3..602430024b6 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java @@ -330,11 +330,34 @@ public Type getElementType() { } public static class OpenSearchRelRunners { + + // ThreadLocal to store the complete RelNode tree for access during scan operations + private static final ThreadLocal CURRENT_REL_NODE = new ThreadLocal<>(); + + public static RelNode getCurrentRelNode() { + return CURRENT_REL_NODE.get(); + } + + private static void setCurrentRelNode(RelNode relNode) { + CURRENT_REL_NODE.set(relNode); + } + + public static void clearCurrentRelNode() { + CURRENT_REL_NODE.remove(); + } + /** * Runs a relational expression by existing connection. This class copied from {@link * org.apache.calcite.tools.RelRunners#run(RelNode)} */ public static PreparedStatement run(CalcitePlanContext context, RelNode rel) { + // Store the complete RelNode tree from context for access during scan operations + // FIXME : This also captures the things which SQL plugin doesn't push down + // FIXME : We need to capture the Calcite when SQL plugin generates DSL instead. + if (context.getCompleteRelNodeTree() != null) { + setCurrentRelNode(context.getCompleteRelNodeTree()); + } + final RelShuttle shuttle = new RelHomogeneousShuttle() { @Override diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteUtils.java index e995d7efd52..a76fa4a39de 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteUtils.java @@ -7,7 +7,14 @@ import static org.opensearch.sql.common.setting.Settings.Key.CALCITE_ENGINE_ENABLED; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; +import java.util.stream.Collectors; import lombok.experimental.UtilityClass; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; @UtilityClass public class CalciteUtils { @@ -16,4 +23,10 @@ public static UnsupportedOperationException getOnlyForCalciteException(String fe return new UnsupportedOperationException( feature + " is supported only when " + CALCITE_ENGINE_ENABLED.getKeyValue() + "=true"); } + + public static Pair, List> partition( + Collection collection, Predicate predicate) { + Map> map = collection.stream().collect(Collectors.partitioningBy(predicate)); + return new ImmutablePair<>(map.get(true), map.get(false)); + } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java index cf231401fd1..848ab118367 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java @@ -143,6 +143,8 @@ public static RelDataType convertExprTypeToRelDataType(ExprType field) { return convertExprTypeToRelDataType(field, true); } + //TODO I think this should be in the interface/abstract depending on the field type and engine type, basically this will come from mapperservice + // Since these fields are UDTs, commented since substrait don't know how to convert these. default making them to BIGINT so that we can bypass these /** Converts a OpenSearch ExprCoreType field to relational type. */ public static RelDataType convertExprTypeToRelDataType(ExprType fieldType, boolean nullable) { if (fieldType instanceof ExprCoreType) { @@ -162,26 +164,30 @@ public static RelDataType convertExprTypeToRelDataType(ExprType fieldType, boole case DOUBLE: return TYPE_FACTORY.createSqlType(SqlTypeName.DOUBLE, nullable); case IP: - return TYPE_FACTORY.createUDT(ExprUDT.EXPR_IP, nullable); + return TYPE_FACTORY.createSqlType(SqlTypeName.BIGINT, nullable); +// return TYPE_FACTORY.createUDT(ExprUDT.EXPR_IP, nullable); case STRING: return TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR, nullable); case BOOLEAN: return TYPE_FACTORY.createSqlType(SqlTypeName.BOOLEAN, nullable); case DATE: - return TYPE_FACTORY.createUDT(ExprUDT.EXPR_DATE, nullable); + // default making them to BIGINT so that we can bypass these + return TYPE_FACTORY.createUDT(ExprUDT.EXPR_DATE, nullable); +// return TYPE_FACTORY.createSqlType(SqlTypeName.DATE, nullable); case TIME: return TYPE_FACTORY.createUDT(ExprUDT.EXPR_TIME, nullable); +// return TYPE_FACTORY.createSqlType(SqlTypeName.BIGINT, nullable); case TIMESTAMP: return TYPE_FACTORY.createUDT(ExprUDT.EXPR_TIMESTAMP, nullable); +// return TYPE_FACTORY.createSqlType(SqlTypeName.TIMESTAMP, 3); case ARRAY: return TYPE_FACTORY.createArrayType( TYPE_FACTORY.createSqlType(SqlTypeName.ANY, nullable), -1); case STRUCT: - // TODO: should use RelRecordType instead of MapSqlType here - // https://github.com/opensearch-project/sql/issues/3459 final RelDataType relKey = TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR); + // TODO: should we provide more precise type here? return TYPE_FACTORY.createMapType( - relKey, TYPE_FACTORY.createSqlType(SqlTypeName.BINARY), nullable); + relKey, TYPE_FACTORY.createSqlType(SqlTypeName.ANY), nullable); case UNKNOWN: default: throw new IllegalArgumentException( @@ -189,19 +195,24 @@ public static RelDataType convertExprTypeToRelDataType(ExprType fieldType, boole } } else { if (fieldType.legacyTypeName().equalsIgnoreCase("binary")) { - return TYPE_FACTORY.createUDT(ExprUDT.EXPR_BINARY, nullable); + return TYPE_FACTORY.createSqlType(SqlTypeName.BIGINT, nullable); +// return TYPE_FACTORY.createUDT(ExprUDT.EXPR_BINARY, nullable); } else if (fieldType.legacyTypeName().equalsIgnoreCase("timestamp")) { return TYPE_FACTORY.createUDT(ExprUDT.EXPR_TIMESTAMP, nullable); +// return TYPE_FACTORY.createSqlType(SqlTypeName.TIMESTAMP, 3); } else if (fieldType.legacyTypeName().equalsIgnoreCase("date")) { - return TYPE_FACTORY.createUDT(ExprUDT.EXPR_DATE, nullable); + return TYPE_FACTORY.createUDT(ExprUDT.EXPR_TIME, nullable); +// return TYPE_FACTORY.createSqlType(SqlTypeName.DATE, nullable); } else if (fieldType.legacyTypeName().equalsIgnoreCase("time")) { return TYPE_FACTORY.createUDT(ExprUDT.EXPR_TIME, nullable); +// return TYPE_FACTORY.createSqlType(SqlTypeName.BIGINT, nullable); } else if (fieldType.legacyTypeName().equalsIgnoreCase("geo_point")) { return TYPE_FACTORY.createSqlType(SqlTypeName.GEOMETRY, nullable); } else if (fieldType.legacyTypeName().equalsIgnoreCase("text")) { return TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR, nullable); } else if (fieldType.legacyTypeName().equalsIgnoreCase("ip")) { - return TYPE_FACTORY.createUDT(ExprUDT.EXPR_IP, nullable); + return TYPE_FACTORY.createSqlType(SqlTypeName.BIGINT, nullable); +// return TYPE_FACTORY.createUDT(ExprUDT.EXPR_IP, nullable); } else if (fieldType.getOriginalPath().isPresent()) { return convertExprTypeToRelDataType(fieldType.getOriginalExprType(), nullable); } else { @@ -311,7 +322,8 @@ public static RelDataType convertSchema(Table table) { List fieldNameList = new ArrayList<>(); List typeList = new ArrayList<>(); Map fieldTypes = new LinkedHashMap<>(table.getFieldTypes()); - fieldTypes.putAll(table.getReservedFieldTypes()); + //reason: We don't need metadata fields in the substrait plan +// fieldTypes.putAll(table.getReservedFieldTypes()); for (Entry entry : fieldTypes.entrySet()) { fieldNameList.add(entry.getKey()); typeList.add(OpenSearchTypeFactory.convertExprTypeToRelDataType(entry.getValue())); @@ -337,4 +349,68 @@ public Type getJavaClass(RelDataType type) { public static boolean isUserDefinedType(RelDataType type) { return type instanceof AbstractExprRelDataType; } + + /** + * Checks if the RelDataType represents a numeric field. Supports both standard SQL numeric types + * (INTEGER, BIGINT, SMALLINT, TINYINT, FLOAT, DOUBLE, DECIMAL, REAL) and OpenSearch UDT numeric + * types. + * + * @param fieldType the RelDataType to check + * @return true if the type is numeric, false otherwise + */ + public static boolean isNumericType(RelDataType fieldType) { + // Check standard SQL numeric types + SqlTypeName sqlType = fieldType.getSqlTypeName(); + if (sqlType == SqlTypeName.INTEGER + || sqlType == SqlTypeName.BIGINT + || sqlType == SqlTypeName.SMALLINT + || sqlType == SqlTypeName.TINYINT + || sqlType == SqlTypeName.FLOAT + || sqlType == SqlTypeName.DOUBLE + || sqlType == SqlTypeName.DECIMAL + || sqlType == SqlTypeName.REAL) { + return true; + } + + // Check for OpenSearch UDT numeric types + if (isUserDefinedType(fieldType)) { + AbstractExprRelDataType exprType = (AbstractExprRelDataType) fieldType; + ExprType udtType = exprType.getExprType(); + return ExprCoreType.numberTypes().contains(udtType); + } + + return false; + } + + /** + * Checks if the RelDataType represents a time-based field (timestamp, date, or time). Supports + * both standard SQL time types (including TIMESTAMP, TIMESTAMP_WITH_LOCAL_TIME_ZONE, DATE, TIME, + * and their timezone variants) and OpenSearch UDT time types. + * + * @param fieldType the RelDataType to check + * @return true if the type is time-based, false otherwise + */ + public static boolean isTimeBasedType(RelDataType fieldType) { + // Check standard SQL time types + SqlTypeName sqlType = fieldType.getSqlTypeName(); + if (sqlType == SqlTypeName.TIMESTAMP + || sqlType == SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE + || sqlType == SqlTypeName.DATE + || sqlType == SqlTypeName.TIME + || sqlType == SqlTypeName.TIME_WITH_LOCAL_TIME_ZONE) { + return true; + } + + // Check for OpenSearch UDT types (EXPR_TIMESTAMP mapped to VARCHAR) + if (isUserDefinedType(fieldType)) { + AbstractExprRelDataType exprType = (AbstractExprRelDataType) fieldType; + ExprType udtType = exprType.getExprType(); + return udtType == ExprCoreType.TIMESTAMP + || udtType == ExprCoreType.DATE + || udtType == ExprCoreType.TIME; + } + + // Fallback check if type string contains EXPR_TIMESTAMP + return fieldType.toString().contains("EXPR_TIMESTAMP"); + } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java index 4d3bef062fa..fefab6d57ce 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java @@ -12,9 +12,12 @@ import static org.apache.calcite.rex.RexWindowBounds.preceding; import com.google.common.collect.ImmutableList; +import java.lang.reflect.Method; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -22,10 +25,14 @@ import org.apache.calcite.rel.RelHomogeneousShuttle; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelShuttle; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rel.logical.LogicalSort; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexCorrelVariable; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexOver; @@ -36,6 +43,7 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; import org.opensearch.sql.ast.AbstractNodeVisitor; import org.opensearch.sql.ast.Node; @@ -54,13 +62,15 @@ public interface PlanUtils { /** this is only for dedup command, do not reuse it in other command */ String ROW_NUMBER_COLUMN_FOR_DEDUP = "_row_number_dedup_"; - String ROW_NUMBER_COLUMN_NAME = "_row_number_"; - String ROW_NUMBER_COLUMN_NAME_MAIN = "_row_number_main_"; - String ROW_NUMBER_COLUMN_NAME_SUBSEARCH = "_row_number_subsearch_"; + String ROW_NUMBER_COLUMN_FOR_RARE_TOP = "_row_number_rare_top_"; + String ROW_NUMBER_COLUMN_FOR_MAIN = "_row_number_main_"; + String ROW_NUMBER_COLUMN_FOR_SUBSEARCH = "_row_number_subsearch_"; + String ROW_NUMBER_COLUMN_FOR_STREAMSTATS = "__stream_seq__"; static SpanUnit intervalUnitToSpanUnit(IntervalUnit unit) { return switch (unit) { - case MICROSECOND -> SpanUnit.MILLISECOND; + case MICROSECOND -> SpanUnit.MICROSECOND; + case MILLISECOND -> SpanUnit.MILLISECOND; case SECOND -> SpanUnit.SECOND; case MINUTE -> SpanUnit.MINUTE; case HOUR -> SpanUnit.HOUR; @@ -74,6 +84,62 @@ static SpanUnit intervalUnitToSpanUnit(IntervalUnit unit) { }; } + static IntervalUnit spanUnitToIntervalUnit(SpanUnit unit) { + switch (unit) { + case MICROSECOND: + case US: + return IntervalUnit.MICROSECOND; + case MILLISECOND: + case MS: + return IntervalUnit.MILLISECOND; + case SECOND: + case SECONDS: + case SEC: + case SECS: + case S: + return IntervalUnit.SECOND; + case MINUTE: + case MINUTES: + case MIN: + case MINS: + case m: + return IntervalUnit.MINUTE; + case HOUR: + case HOURS: + case HR: + case HRS: + case H: + return IntervalUnit.HOUR; + case DAY: + case DAYS: + case D: + return IntervalUnit.DAY; + case WEEK: + case WEEKS: + case W: + return IntervalUnit.WEEK; + case MONTH: + case MONTHS: + case MON: + case M: + return IntervalUnit.MONTH; + case QUARTER: + case QUARTERS: + case QTR: + case QTRS: + case Q: + return IntervalUnit.QUARTER; + case YEAR: + case YEARS: + case Y: + return IntervalUnit.YEAR; + case UNKNOWN: + return IntervalUnit.UNKNOWN; + default: + throw new UnsupportedOperationException("Unsupported span unit: " + unit); + } + } + static RexNode makeOver( CalcitePlanContext context, BuiltinFunctionName functionName, @@ -382,10 +448,18 @@ static RexNode derefMapCall(RexNode rexNode) { return rexNode; } - /** Check if contains RexOver */ + /** Check if contains RexOver introduced by dedup */ static boolean containsRowNumberDedup(LogicalProject project) { return project.getProjects().stream() - .anyMatch(p -> p instanceof RexOver && p.getKind() == SqlKind.ROW_NUMBER); + .anyMatch(p -> p instanceof RexOver && p.getKind() == SqlKind.ROW_NUMBER) + && project.getRowType().getFieldNames().contains(ROW_NUMBER_COLUMN_FOR_DEDUP); + } + + /** Check if contains RexOver introduced by dedup top/rare */ + static boolean containsRowNumberRareTop(LogicalProject project) { + return project.getProjects().stream() + .anyMatch(p -> p instanceof RexOver && p.getKind() == SqlKind.ROW_NUMBER) + && project.getRowType().getFieldNames().contains(ROW_NUMBER_COLUMN_FOR_RARE_TOP); } /** Get all RexWindow list from LogicalProject */ @@ -419,13 +493,51 @@ public Void visitInputRef(RexInputRef inputRef) { return selectedColumns; } + // `RelDecorrelator` may generate a Project with duplicated fields, e.g. Project($0,$0). + // There will be problem if pushing down the pattern like `Aggregate(AGG($0),{1})-Project($0,$0)`, + // as it will lead to field-name conflict. + // We should wait and rely on `AggregateProjectMergeRule` to mitigate it by having this constraint + // Nevertheless, that rule cannot handle all cases if there is RexCall in the Project, + // e.g. Project($0, $0, +($0,1)). We cannot push down the Aggregate for this corner case. + // TODO: Simplify the Project where there is RexCall by adding a new rule. + static boolean distinctProjectList(LogicalProject project) { + // Change to Set> to resolve + // https://github.com/opensearch-project/sql/issues/4347 + Set> rexSet = new HashSet<>(); + return project.getNamedProjects().stream().allMatch(rexSet::add); + } + + static boolean containsRexOver(LogicalProject project) { + return project.getProjects().stream().anyMatch(RexOver::containsOver); + } + + /** + * The LogicalSort is a LIMIT that should be pushed down when its fetch field is not null and its + * collation is empty. For example: sort name | head 5 should not be pushed down + * because it has a field collation. + * + * @param sort The LogicalSort to check. + * @return True if the LogicalSort is a LIMIT, false otherwise. + */ + static boolean isLogicalSortLimit(LogicalSort sort) { + return sort.fetch != null; + } + + static boolean projectContainsExpr(Project project) { + return project.getProjects().stream().anyMatch(p -> p instanceof RexCall); + } + + static boolean sortByFieldsOnly(Sort sort) { + return !sort.getCollation().getFieldCollations().isEmpty() && sort.fetch == null; + } + /** * Get a string representation of the argument types expressed in ExprType for error messages. * * @param argTypes the list of argument types as {@link RelDataType} * @return a string in the format [type1,type2,...] representing the argument types */ - public static String getActualSignature(List argTypes) { + static String getActualSignature(List argTypes) { return "[" + argTypes.stream() .map(OpenSearchTypeFactory::convertRelDataTypeToExprType) @@ -433,4 +545,36 @@ public static String getActualSignature(List argTypes) { .collect(Collectors.joining(",")) + "]"; } + + /** + * Check if the RexNode contains any CorrelVariable. + * + * @param node the RexNode to check + * @return true if the RexNode contains any CorrelVariable, false otherwise + */ + static boolean containsCorrelVariable(RexNode node) { + try { + node.accept( + new RexVisitorImpl(true) { + @Override + public Void visitCorrelVariable(RexCorrelVariable correlVar) { + throw new RuntimeException("Correl found"); + } + }); + return false; + } catch (Exception e) { + return true; + } + } + + /** Adds a rel node to the top of the stack while preserving the field names and aliases. */ + static void replaceTop(RelBuilder relBuilder, RelNode relNode) { + try { + Method method = RelBuilder.class.getDeclaredMethod("replaceTop", RelNode.class); + method.setAccessible(true); + method.invoke(relBuilder, relNode); + } catch (Exception e) { + throw new IllegalStateException("Unable to invoke RelBuilder.replaceTop", e); + } + } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/SubsearchUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/SubsearchUtils.java new file mode 100644 index 00000000000..221e4275f64 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/SubsearchUtils.java @@ -0,0 +1,107 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.utils; + +import java.util.List; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.experimental.UtilityClass; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelShuttleImpl; +import org.apache.calcite.rel.logical.LogicalCorrelate; +import org.apache.calcite.rel.logical.LogicalFilter; +import org.apache.calcite.rel.logical.LogicalIntersect; +import org.apache.calcite.rel.logical.LogicalJoin; +import org.apache.calcite.rel.logical.LogicalMinus; +import org.apache.calcite.rel.logical.LogicalUnion; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.calcite.plan.LogicalSystemLimit; + +@UtilityClass +public class SubsearchUtils { + + /** Insert a system_limit under correlate conditions. */ + private static RelNode insertSysLimitUnderCorrelateConditions( + LogicalFilter logicalFilter, CalcitePlanContext context) { + // Before: + // LogicalFilter(condition=[AND(=($cor0.SAL, $2), >($1, 1000.0:DECIMAL(5, 1)))]) + // After: + // LogicalFilter(condition=[=($cor0.SAL, $2)]) + // LogicalSystemLimit(fetch=[1], type=[SUBSEARCH_MAXOUT]) + // LogicalFilter(condition=[>($1, 1000.0:DECIMAL(5, 1))]) + RexNode originalCondition = logicalFilter.getCondition(); + List conditions = RelOptUtil.conjunctions(originalCondition); + Pair, List> result = + CalciteUtils.partition(conditions, PlanUtils::containsCorrelVariable); + if (result.getLeft().isEmpty()) { + return logicalFilter; + } + + RelNode input = logicalFilter.getInput(); + if (!result.getRight().isEmpty()) { + RexNode nonCorrelCondition = + RexUtil.composeConjunction(context.rexBuilder, result.getRight()); + input = LogicalFilter.create(input, nonCorrelCondition); + } + input = + LogicalSystemLimit.create( + LogicalSystemLimit.SystemLimitType.SUBSEARCH_MAXOUT, + input, + context.relBuilder.literal(context.sysLimit.subsearchLimit())); + if (!result.getLeft().isEmpty()) { + RexNode correlCondition = RexUtil.composeConjunction(context.rexBuilder, result.getLeft()); + input = LogicalFilter.create(input, correlCondition); + } + return input; + } + + /** Insert a system_limit under correlated conditions by visiting a plan tree. */ + @RequiredArgsConstructor + public static class SystemLimitInsertionShuttle extends RelShuttleImpl { + + private final CalcitePlanContext context; + @Getter private boolean correlatedConditionFound = false; + + @Override + public RelNode visit(LogicalFilter filter) { + RelNode newFilter = insertSysLimitUnderCorrelateConditions(filter, context); + if (newFilter != filter) { + correlatedConditionFound = true; + return newFilter; + } + return super.visitChildren(filter); + } + + @Override + public RelNode visit(LogicalJoin node) { + return node; + } + + @Override + public RelNode visit(LogicalCorrelate node) { + return node; + } + + @Override + public RelNode visit(LogicalUnion node) { + return node; + } + + @Override + public RelNode visit(LogicalIntersect node) { + return node; + } + + @Override + public RelNode visit(LogicalMinus node) { + return node; + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java index e7cee54c87e..32aee242388 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java @@ -178,7 +178,13 @@ public static List convertToExprValues( types.stream().map(OpenSearchTypeFactory::convertRelDataTypeToExprType).toList(); List exprValues = new ArrayList<>(); for (int i = 0; i < operands.size(); i++) { - Expression operand = Expressions.convert_(operands.get(i), Object.class); + // TODO a workaround of Apache Calcite bug in 1.41.0: + // If you call Expressions.convert_(expr, Number.class) or + // Expressions.convert_(expr, Object.class), + // you must change to Expressions.convert_(Expressions.box(expr), Number.class/Object.class). + // Because the codegen in Janino.UnitCompiler, "(Object) -1" will be mistakenly treated to + // "Object subtracting one" instead of "type casting on native one". + Expression operand = Expressions.convert_(Expressions.box(operands.get(i)), Object.class); exprValues.add( i, Expressions.call( diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/WildcardUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/WildcardUtils.java index 09552e97109..8558a5292b7 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/WildcardUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/WildcardUtils.java @@ -5,6 +5,7 @@ package org.opensearch.sql.calcite.utils; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -92,4 +93,141 @@ private static boolean matchesCompiledPattern(String[] parts, String fieldName) public static boolean containsWildcard(String str) { return str != null && str.contains(WILDCARD); } + + /** + * Converts a wildcard pattern to a regex pattern. + * + *

Example: "*ada" → "^(.*?)ada$" + * + * @param wildcardPattern wildcard pattern with '*' and escape sequences (\*, \\) + * @return regex pattern with capture groups + */ + public static String convertWildcardPatternToRegex(String wildcardPattern) { + String[] parts = splitWildcards(wildcardPattern); + StringBuilder regexBuilder = new StringBuilder("^"); + + for (int i = 0; i < parts.length; i++) { + regexBuilder.append(java.util.regex.Pattern.quote(parts[i])); + if (i < parts.length - 1) { + regexBuilder.append("(.*?)"); // Non-greedy capture group for wildcard + } + } + regexBuilder.append("$"); + + return regexBuilder.toString(); + } + + /** + * Converts a wildcard replacement string to a regex replacement string. + * + *

Example: "*_*" → "$1_$2" + * + * @param wildcardReplacement replacement string with '*' and escape sequences (\*, \\) + * @return regex replacement string with capture group references + */ + public static String convertWildcardReplacementToRegex(String wildcardReplacement) { + if (!wildcardReplacement.contains("*")) { + return wildcardReplacement; // No wildcards = literal replacement + } + + StringBuilder result = new StringBuilder(); + int captureIndex = 1; // Regex capture groups start at $1 + boolean escaped = false; + + for (char c : wildcardReplacement.toCharArray()) { + if (escaped) { + // Handle escape sequences: \* or \\ + result.append(c); + escaped = false; + } else if (c == '\\') { + escaped = true; + } else if (c == '*') { + // Replace wildcard with $1, $2, etc. + result.append('$').append(captureIndex++); + } else { + result.append(c); + } + } + + return result.toString(); + } + + /** + * Splits a wildcard pattern into parts separated by unescaped wildcards. + * + *

Example: "a*b*c" → ["a", "b", "c"] + * + * @param pattern wildcard pattern with escape sequences + * @return array of pattern parts + */ + private static String[] splitWildcards(String pattern) { + List parts = new ArrayList<>(); + StringBuilder current = new StringBuilder(); + boolean escaped = false; + + for (char c : pattern.toCharArray()) { + if (escaped) { + current.append(c); + escaped = false; + } else if (c == '\\') { + escaped = true; + } else if (c == '*') { + parts.add(current.toString()); + current = new StringBuilder(); + } else { + current.append(c); + } + } + + if (escaped) { + throw new IllegalArgumentException( + "Invalid escape sequence: pattern ends with unescaped backslash"); + } + + parts.add(current.toString()); + return parts.toArray(new String[0]); + } + + /** + * Counts the number of unescaped wildcards in a string. + * + * @param str string to count wildcards in + * @return number of unescaped wildcards + */ + private static int countWildcards(String str) { + int count = 0; + boolean escaped = false; + for (char c : str.toCharArray()) { + if (escaped) { + escaped = false; + } else if (c == '\\') { + escaped = true; + } else if (c == '*') { + count++; + } + } + return count; + } + + /** + * Validates that wildcard count is symmetric between pattern and replacement. + * + *

Replacement must have either the same number of wildcards as the pattern, or zero wildcards. + * + * @param pattern wildcard pattern + * @param replacement wildcard replacement + * @throws IllegalArgumentException if wildcard counts are mismatched + */ + public static void validateWildcardSymmetry(String pattern, String replacement) { + int patternWildcards = countWildcards(pattern); + int replacementWildcards = countWildcards(replacement); + + if (replacementWildcards != 0 && replacementWildcards != patternWildcards) { + throw new IllegalArgumentException( + String.format( + "Error in 'replace' command: Wildcard count mismatch - pattern has %d wildcard(s), " + + "replacement has %d. Replacement must have same number of wildcards or none.", + patternWildcards, replacementWildcards)); + } + } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/BinFieldValidator.java b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/BinFieldValidator.java index 1396b12dc97..3bdbad694a7 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/BinFieldValidator.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/BinFieldValidator.java @@ -6,16 +6,11 @@ package org.opensearch.sql.calcite.utils.binning; import java.util.List; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.ast.expression.Field; import org.opensearch.sql.ast.tree.Bin; import org.opensearch.sql.calcite.CalcitePlanContext; -import org.opensearch.sql.calcite.type.AbstractExprRelDataType; -import org.opensearch.sql.data.type.ExprCoreType; -import org.opensearch.sql.data.type.ExprType; -/** Utility class for field validation and type checking in bin operations. */ +/** Utility class for bin-specific field operations. */ public class BinFieldValidator { /** Extracts the field name from a Bin node. */ @@ -37,26 +32,4 @@ public static void validateFieldExists(String fieldName, CalcitePlanContext cont "Field '%s' not found in dataset. Available fields: %s", fieldName, availableFields)); } } - - /** Checks if the field type is time-based. */ - public static boolean isTimeBasedField(RelDataType fieldType) { - // Check standard SQL time types - SqlTypeName sqlType = fieldType.getSqlTypeName(); - if (sqlType == SqlTypeName.TIMESTAMP - || sqlType == SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE - || sqlType == SqlTypeName.DATE) { - return true; - } - - // Check for OpenSearch UDT types (EXPR_TIMESTAMP mapped to VARCHAR) - if (fieldType instanceof AbstractExprRelDataType exprType) { - ExprType udtType = exprType.getExprType(); - return udtType == ExprCoreType.TIMESTAMP - || udtType == ExprCoreType.DATE - || udtType == ExprCoreType.TIME; - } - - // Check if type string contains EXPR_TIMESTAMP - return fieldType.toString().contains("EXPR_TIMESTAMP"); - } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/BinnableField.java b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/BinnableField.java new file mode 100644 index 00000000000..c8c73ce3a99 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/BinnableField.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.utils.binning; + +import lombok.Getter; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexNode; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.exception.SemanticCheckException; + +/** + * Represents a validated field that supports binning operations. The existence of this class + * guarantees that validation has been run - the field is either numeric or time-based. + * + *

This design encodes validation in the type system, preventing downstream code from forgetting + * to validate or running validation multiple times. + */ +@Getter +public class BinnableField { + private final RexNode fieldExpr; + private final RelDataType fieldType; + private final String fieldName; + private final boolean isTimeBased; + private final boolean isNumeric; + + /** + * Creates a validated BinnableField. Throws SemanticCheckException if the field is neither + * numeric nor time-based. + * + * @param fieldExpr The Rex expression for the field + * @param fieldType The relational data type of the field + * @param fieldName The name of the field (for error messages) + * @throws SemanticCheckException if the field is neither numeric nor time-based + */ + public BinnableField(RexNode fieldExpr, RelDataType fieldType, String fieldName) { + this.fieldExpr = fieldExpr; + this.fieldType = fieldType; + this.fieldName = fieldName; + + this.isTimeBased = OpenSearchTypeFactory.isTimeBasedType(fieldType); + this.isNumeric = OpenSearchTypeFactory.isNumericType(fieldType); + + // Validation: field must be either numeric or time-based + if (!isNumeric && !isTimeBased) { + throw new SemanticCheckException( + String.format( + "Cannot apply binning: field '%s' is non-numeric and not time-related, expected" + + " numeric or time-related type", + fieldName)); + } + } + + /** + * Returns true if this field requires numeric binning logic (not time-based binning). + * + * @return true if the field should use numeric binning, false if it should use time-based binning + */ + public boolean requiresNumericBinning() { + return !isTimeBased; + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/CountBinHandler.java b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/CountBinHandler.java index 9716eab993c..7422a26f0b7 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/CountBinHandler.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/CountBinHandler.java @@ -13,7 +13,9 @@ import org.opensearch.sql.calcite.CalcitePlanContext; import org.opensearch.sql.calcite.CalciteRexNodeVisitor; import org.opensearch.sql.calcite.utils.binning.BinConstants; +import org.opensearch.sql.calcite.utils.binning.BinFieldValidator; import org.opensearch.sql.calcite.utils.binning.BinHandler; +import org.opensearch.sql.calcite.utils.binning.BinnableField; import org.opensearch.sql.expression.function.PPLBuiltinOperators; /** Handler for bins-based (count) binning operations. */ @@ -25,6 +27,11 @@ public RexNode createExpression( CountBin countBin = (CountBin) node; + // Create validated binnable field (validates that field is numeric or time-based) + // Note: bins parameter works with both numeric and time-based fields + String fieldName = BinFieldValidator.extractFieldName(node); + BinnableField field = new BinnableField(fieldExpr, fieldExpr.getType(), fieldName); + Integer requestedBins = countBin.getBins(); if (requestedBins == null) { requestedBins = BinConstants.DEFAULT_BINS; diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/DefaultBinHandler.java b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/DefaultBinHandler.java index b022df03b79..e68477a9566 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/DefaultBinHandler.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/DefaultBinHandler.java @@ -5,7 +5,6 @@ package org.opensearch.sql.calcite.utils.binning.handlers; -import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.opensearch.sql.ast.tree.Bin; @@ -15,6 +14,7 @@ import org.opensearch.sql.calcite.utils.BinTimeSpanUtils; import org.opensearch.sql.calcite.utils.binning.BinFieldValidator; import org.opensearch.sql.calcite.utils.binning.BinHandler; +import org.opensearch.sql.calcite.utils.binning.BinnableField; import org.opensearch.sql.calcite.utils.binning.RangeFormatter; /** Handler for default binning when no parameters are specified. */ @@ -25,11 +25,13 @@ public RexNode createExpression( Bin node, RexNode fieldExpr, CalcitePlanContext context, CalciteRexNodeVisitor visitor) { DefaultBin defaultBin = (DefaultBin) node; - RelDataType fieldType = fieldExpr.getType(); String fieldName = BinFieldValidator.extractFieldName(node); + // Create validated binnable field (validates that field is numeric or time-based) + BinnableField field = new BinnableField(fieldExpr, fieldExpr.getType(), fieldName); + // Use time-based binning for time fields - if (BinFieldValidator.isTimeBasedField(fieldType)) { + if (field.isTimeBased()) { BinFieldValidator.validateFieldExists(fieldName, context); return BinTimeSpanUtils.createBinTimeSpanExpression(fieldExpr, 1, "h", 0, context); } diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/MinSpanBinHandler.java b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/MinSpanBinHandler.java index 31e3c11d243..16e11b7abce 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/MinSpanBinHandler.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/MinSpanBinHandler.java @@ -15,7 +15,9 @@ import org.opensearch.sql.ast.tree.MinSpanBin; import org.opensearch.sql.calcite.CalcitePlanContext; import org.opensearch.sql.calcite.CalciteRexNodeVisitor; +import org.opensearch.sql.calcite.utils.binning.BinFieldValidator; import org.opensearch.sql.calcite.utils.binning.BinHandler; +import org.opensearch.sql.calcite.utils.binning.BinnableField; import org.opensearch.sql.expression.function.PPLBuiltinOperators; /** Handler for minspan-based binning operations. */ @@ -27,6 +29,16 @@ public RexNode createExpression( MinSpanBin minSpanBin = (MinSpanBin) node; + // Create validated binnable field (validates that field is numeric or time-based) + String fieldName = BinFieldValidator.extractFieldName(node); + BinnableField field = new BinnableField(fieldExpr, fieldExpr.getType(), fieldName); + + // Minspan binning requires numeric fields + if (!field.requiresNumericBinning()) { + throw new IllegalArgumentException( + "Minspan binning is only supported for numeric fields, not time-based fields"); + } + RexNode minspanValue = visitor.analyze(minSpanBin.getMinspan(), context); if (!minspanValue.isA(LITERAL)) { diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/RangeBinHandler.java b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/RangeBinHandler.java index 85e2b701528..aa726cb9dbb 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/RangeBinHandler.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/RangeBinHandler.java @@ -10,7 +10,9 @@ import org.opensearch.sql.ast.tree.RangeBin; import org.opensearch.sql.calcite.CalcitePlanContext; import org.opensearch.sql.calcite.CalciteRexNodeVisitor; +import org.opensearch.sql.calcite.utils.binning.BinFieldValidator; import org.opensearch.sql.calcite.utils.binning.BinHandler; +import org.opensearch.sql.calcite.utils.binning.BinnableField; import org.opensearch.sql.expression.function.PPLBuiltinOperators; /** Handler for range-based binning (start/end parameters only). */ @@ -22,6 +24,16 @@ public RexNode createExpression( RangeBin rangeBin = (RangeBin) node; + // Create validated binnable field (validates that field is numeric or time-based) + String fieldName = BinFieldValidator.extractFieldName(node); + BinnableField field = new BinnableField(fieldExpr, fieldExpr.getType(), fieldName); + + // Range binning requires numeric fields + if (!field.requiresNumericBinning()) { + throw new IllegalArgumentException( + "Range binning (start/end) is only supported for numeric fields, not time-based fields"); + } + // Simple MIN/MAX calculation - cleaner than complex CASE expressions RexNode dataMin = context.relBuilder.min(fieldExpr).over().toRex(); RexNode dataMax = context.relBuilder.max(fieldExpr).over().toRex(); diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/SpanBinHandler.java b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/SpanBinHandler.java index ba482f8fd61..1548ae3e7c2 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/SpanBinHandler.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/SpanBinHandler.java @@ -29,8 +29,12 @@ public RexNode createExpression( SpanBin spanBin = (SpanBin) node; + // Create validated binnable field (validates that field is numeric or time-based) + String fieldName = BinFieldValidator.extractFieldName(node); + BinnableField field = new BinnableField(fieldExpr, fieldExpr.getType(), fieldName); + // Handle time-based fields - if (BinFieldValidator.isTimeBasedField(fieldExpr.getType())) { + if (field.isTimeBased()) { return handleTimeBasedSpan(spanBin, fieldExpr, context, visitor); } @@ -64,6 +68,7 @@ private RexNode handleTimeBasedSpan( private RexNode handleNumericOrLogSpan( SpanBin node, RexNode fieldExpr, CalcitePlanContext context, CalciteRexNodeVisitor visitor) { + // Field is already validated by createExpression - must be numeric since we're in this branch RexNode spanValue = visitor.analyze(node.getSpan(), context); if (!spanValue.isA(LITERAL)) { diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprTupleValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprTupleValue.java index b92fdb51bf2..d2b8850d386 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprTupleValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprTupleValue.java @@ -14,6 +14,7 @@ import lombok.RequiredArgsConstructor; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.data.utils.ComparableLinkedHashMap; import org.opensearch.sql.storage.bindingtuple.BindingTuple; import org.opensearch.sql.storage.bindingtuple.LazyBindingTuple; @@ -44,7 +45,7 @@ public Object value() { @Override public Object valueForCalcite() { - LinkedHashMap resultMap = new LinkedHashMap<>(); + ComparableLinkedHashMap resultMap = new ComparableLinkedHashMap<>(); for (Entry entry : valueMap.entrySet()) { resultMap.put(entry.getKey(), entry.getValue().valueForCalcite()); } diff --git a/core/src/main/java/org/opensearch/sql/data/utils/ComparableLinkedHashMap.java b/core/src/main/java/org/opensearch/sql/data/utils/ComparableLinkedHashMap.java new file mode 100644 index 00000000000..31a849dec9b --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/data/utils/ComparableLinkedHashMap.java @@ -0,0 +1,76 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.data.utils; + +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; + +public class ComparableLinkedHashMap extends LinkedHashMap + implements Comparable> { + + public ComparableLinkedHashMap() { + super(); + } + + public ComparableLinkedHashMap(int initialCapacity) { + super(initialCapacity); + } + + public ComparableLinkedHashMap(int initialCapacity, float loadFactor) { + super(initialCapacity, loadFactor); + } + + @Override + public int compareTo(ComparableLinkedHashMap other) { + if (this.isEmpty() && other.isEmpty()) return 0; + if (this.isEmpty()) return -1; + if (other.isEmpty()) return 1; + Iterator> thisIterator = this.entrySet().iterator(); + Iterator> otherIterator = other.entrySet().iterator(); + return compareRecursive(thisIterator, otherIterator); + } + + private int compareRecursive( + Iterator> thisIterator, Iterator> otherIterator) { + boolean thisHasNext = thisIterator.hasNext(); + boolean otherHasNext = otherIterator.hasNext(); + if (!thisHasNext && !otherHasNext) return 0; + if (!thisHasNext) return -1; + if (!otherHasNext) return 1; + + Map.Entry thisEntry = thisIterator.next(); + Map.Entry otherEntry = otherIterator.next(); + K thisKey = thisEntry.getKey(); + K otherKey = otherEntry.getKey(); + V thisValue = thisEntry.getValue(); + V otherValue = otherEntry.getValue(); + int comparison = compareKV(thisKey, otherKey, thisValue, otherValue); + if (comparison != 0) return comparison; + return compareRecursive(thisIterator, otherIterator); + } + + @SuppressWarnings("unchecked") + private int compareKV(K key1, K key2, V value1, V value2) { + int keyCompare; + if (key1 instanceof Comparable) { + keyCompare = ((Comparable) key1).compareTo(key2); + } else { + keyCompare = key1.toString().compareTo(key2.toString()); + } + if (keyCompare != 0) { + return keyCompare; + } + + if (value1 == null && value2 == null) return 0; + if (value1 == null) return -1; + if (value2 == null) return 1; + if (value1 instanceof Comparable) { + return ((Comparable) value1).compareTo(value2); + } + return value1.toString().compareTo(value2.toString()); + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java b/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java index 78b0ead1c8c..ec1e427e365 100644 --- a/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java +++ b/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java @@ -109,6 +109,22 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(root, calcite); } + + public static ExplainResponse normalizeLf(ExplainResponse response) { + ExecutionEngine.ExplainResponseNodeV2 calcite = response.getCalcite(); + if (calcite != null) { + return new ExplainResponse( + new ExecutionEngine.ExplainResponseNodeV2( + normalizeLf(calcite.getLogical()), + normalizeLf(calcite.getPhysical()), + normalizeLf(calcite.getExtended()))); + } + return response; + } + + private static String normalizeLf(String value) { + return value == null ? null : value.replace("\r\n", "\n"); + } } @AllArgsConstructor diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryService.java b/core/src/main/java/org/opensearch/sql/executor/QueryService.java index 995d7d55e0d..5a71a7b74eb 100644 --- a/core/src/main/java/org/opensearch/sql/executor/QueryService.java +++ b/core/src/main/java/org/opensearch/sql/executor/QueryService.java @@ -33,11 +33,11 @@ import org.opensearch.sql.calcite.CalcitePlanContext; import org.opensearch.sql.calcite.CalciteRelNodeVisitor; import org.opensearch.sql.calcite.OpenSearchSchema; +import org.opensearch.sql.calcite.SysLimit; import org.opensearch.sql.calcite.plan.LogicalSystemLimit; import org.opensearch.sql.calcite.plan.LogicalSystemLimit.SystemLimitType; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.common.setting.Settings; -import org.opensearch.sql.common.setting.Settings.Key; import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.exception.CalciteUnsupportedException; import org.opensearch.sql.exception.NonFallbackCalciteException; @@ -90,35 +90,41 @@ public void executeWithCalcite( UnresolvedPlan plan, QueryType queryType, ResponseListener listener) { - try { - AccessController.doPrivileged( - (PrivilegedAction) - () -> { - CalcitePlanContext context = - CalcitePlanContext.create( - buildFrameworkConfig(), getQuerySizeLimit(), queryType); - RelNode relNode = analyze(plan, context); - RelNode optimized = optimize(relNode, context); - RelNode calcitePlan = convertToCalcitePlan(optimized); - executionEngine.execute(calcitePlan, context, listener); - return null; - }); - } catch (Throwable t) { - if (isCalciteFallbackAllowed(t) && !(t instanceof NonFallbackCalciteException)) { - log.warn("Fallback to V2 query engine since got exception", t); - executeWithLegacy(plan, queryType, listener, Optional.of(t)); - } else { - if (t instanceof Exception) { - listener.onFailure((Exception) t); - } else if (t instanceof VirtualMachineError) { - // throw and fast fail the VM errors such as OOM (same with v2). - throw t; - } else { - // Calcite may throw AssertError during query execution. - listener.onFailure(new CalciteUnsupportedException(t.getMessage(), t)); - } - } - } + CalcitePlanContext.run( + () -> { + try { + AccessController.doPrivileged( + (PrivilegedAction) + () -> { + CalcitePlanContext context = + CalcitePlanContext.create( + buildFrameworkConfig(), SysLimit.fromSettings(settings), queryType); + RelNode relNode = analyze(plan, context); + RelNode optimized = optimize(relNode, context); + RelNode calcitePlan = convertToCalcitePlan(optimized); + // Store the complete RelNode tree in context, this is required to convert to substrait during scan + context.setCompleteRelNodeTree(calcitePlan); + executionEngine.execute(calcitePlan, context, listener); + return null; + }); + } catch (Throwable t) { + if (isCalciteFallbackAllowed(t) && !(t instanceof NonFallbackCalciteException)) { + log.warn("Fallback to V2 query engine since got exception", t); + executeWithLegacy(plan, queryType, listener, Optional.of(t)); + } else { + if (t instanceof Exception) { + listener.onFailure((Exception) t); + } else if (t instanceof VirtualMachineError) { + // throw and fast fail the VM errors such as OOM (same with v2). + throw t; + } else { + // Calcite may throw AssertError during query execution. + listener.onFailure(new CalciteUnsupportedException(t.getMessage(), t)); + } + } + } + }, + settings); } public void explainWithCalcite( @@ -126,32 +132,40 @@ public void explainWithCalcite( QueryType queryType, ResponseListener listener, Explain.ExplainFormat format) { - try { - AccessController.doPrivileged( - (PrivilegedAction) - () -> { - CalcitePlanContext context = - CalcitePlanContext.create( - buildFrameworkConfig(), getQuerySizeLimit(), queryType); - RelNode relNode = analyze(plan, context); - RelNode optimized = optimize(relNode, context); - RelNode calcitePlan = convertToCalcitePlan(optimized); - executionEngine.explain(calcitePlan, format, context, listener); - return null; - }); - } catch (Throwable t) { - if (isCalciteFallbackAllowed(t)) { - log.warn("Fallback to V2 query engine since got exception", t); - explainWithLegacy(plan, queryType, listener, format, Optional.of(t)); - } else { - if (t instanceof Error) { - // Calcite may throw AssertError during query execution. - listener.onFailure(new CalciteUnsupportedException(t.getMessage())); - } else { - listener.onFailure((Exception) t); - } - } - } + CalcitePlanContext.run( + () -> { + try { + AccessController.doPrivileged( + (PrivilegedAction) + () -> { + CalcitePlanContext context = + CalcitePlanContext.create( + buildFrameworkConfig(), SysLimit.fromSettings(settings), queryType); + context.run( + () -> { + RelNode relNode = analyze(plan, context); + RelNode optimized = optimize(relNode, context); + RelNode calcitePlan = convertToCalcitePlan(optimized); + executionEngine.explain(calcitePlan, format, context, listener); + }, + settings); + return null; + }); + } catch (Throwable t) { + if (isCalciteFallbackAllowed(t)) { + log.warn("Fallback to V2 query engine since got exception", t); + explainWithLegacy(plan, queryType, listener, format, Optional.of(t)); + } else { + if (t instanceof Error) { + // Calcite may throw AssertError during query execution. + listener.onFailure(new CalciteUnsupportedException(t.getMessage(), t)); + } else { + listener.onFailure((Exception) t); + } + } + } + }, + settings); } public void executeWithLegacy( @@ -189,7 +203,8 @@ public void explainWithLegacy( Explain.ExplainFormat format, Optional calciteFailure) { try { - if (format != null && format != Explain.ExplainFormat.STANDARD) { + if (format != null + && (format != Explain.ExplainFormat.STANDARD && format != Explain.ExplainFormat.YAML)) { throw new UnsupportedOperationException( "Explain mode " + format.name() + " is not supported in v2 engine"); } @@ -230,7 +245,9 @@ public void executePlan( ExecutionContext.querySizeLimit( // For pagination, querySizeLimit shouldn't take effect. // See {@link PaginationWindowIT::testQuerySizeLimitDoesNotEffectPageSize} - plan instanceof LogicalPaginate ? null : getQuerySizeLimit()), + plan instanceof LogicalPaginate + ? null + : SysLimit.fromSettings(settings).querySizeLimit()), listener)); } catch (Exception e) { listener.onFailure(e); @@ -257,7 +274,9 @@ public PhysicalPlan plan(LogicalPlan plan) { */ public RelNode optimize(RelNode plan, CalcitePlanContext context) { return LogicalSystemLimit.create( - SystemLimitType.QUERY_SIZE_LIMIT, plan, context.relBuilder.literal(context.querySizeLimit)); + SystemLimitType.QUERY_SIZE_LIMIT, + plan, + context.relBuilder.literal(context.sysLimit.querySizeLimit())); } private boolean isCalciteFallbackAllowed(@Nullable Throwable t) { @@ -286,14 +305,10 @@ private boolean isCalciteEnabled(Settings settings) { } } - private Integer getQuerySizeLimit() { - return settings == null ? null : settings.getSettingValue(Key.QUERY_SIZE_LIMIT); - } - // TODO https://github.com/opensearch-project/sql/issues/3457 // Calcite is not available for SQL query now. Maybe release in 3.1.0? private boolean shouldUseCalcite(QueryType queryType) { - return isCalciteEnabled(settings) && queryType == QueryType.PPL; + return true;//isCalciteEnabled(settings) && queryType == QueryType.PPL; } private FrameworkConfig buildFrameworkConfig() { diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunctions.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunctions.java index 9e667937d71..f5d3c44b7e1 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunctions.java @@ -8,6 +8,7 @@ import static java.time.temporal.ChronoUnit.DAYS; import static java.time.temporal.ChronoUnit.HOURS; import static java.time.temporal.ChronoUnit.MICROS; +import static java.time.temporal.ChronoUnit.MILLIS; import static java.time.temporal.ChronoUnit.MINUTES; import static java.time.temporal.ChronoUnit.MONTHS; import static java.time.temporal.ChronoUnit.SECONDS; @@ -1896,6 +1897,9 @@ public static ExprValue exprTimestampAdd( case "MICROSECOND": temporalUnit = MICROS; break; + case "MILLISECOND": + temporalUnit = MILLIS; + break; case "SECOND": temporalUnit = SECONDS; break; @@ -1935,10 +1939,13 @@ public static ExprValue exprTimestampAddForTimeType( private ExprValue getTimeDifference(String part, LocalDateTime startTime, LocalDateTime endTime) { long returnVal; - switch (part) { + switch (part.toUpperCase(Locale.ROOT)) { case "MICROSECOND": returnVal = MICROS.between(startTime, endTime); break; + case "MILLISECOND": + returnVal = MILLIS.between(startTime, endTime); + break; case "SECOND": returnVal = SECONDS.between(startTime, endTime); break; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index d5f79b8946c..ced98022ca9 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -62,6 +62,10 @@ public enum BuiltinFunctionName { /** Collection functions */ ARRAY(FunctionName.of("array")), ARRAY_LENGTH(FunctionName.of("array_length")), + MAP_APPEND(FunctionName.of("map_append"), true), + MAP_CONCAT(FunctionName.of("map_concat"), true), + MAP_REMOVE(FunctionName.of("map_remove"), true), + MVAPPEND(FunctionName.of("mvappend")), MVJOIN(FunctionName.of("mvjoin")), FORALL(FunctionName.of("forall")), EXISTS(FunctionName.of("exists")), @@ -246,6 +250,7 @@ public enum BuiltinFunctionName { JSON_ARRAY(FunctionName.of("json_array")), JSON_ARRAY_LENGTH(FunctionName.of("json_array_length")), JSON_EXTRACT(FunctionName.of("json_extract")), + JSON_EXTRACT_ALL(FunctionName.of("json_extract_all"), true), JSON_KEYS(FunctionName.of("json_keys")), JSON_SET(FunctionName.of("json_set")), JSON_DELETE(FunctionName.of("json_delete")), diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CoercionUtils.java b/core/src/main/java/org/opensearch/sql/expression/function/CoercionUtils.java index f0c6fc84837..ce78d6dec21 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CoercionUtils.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CoercionUtils.java @@ -5,19 +5,27 @@ package org.opensearch.sql.expression.function; +import static org.opensearch.sql.data.type.ExprCoreType.UNKNOWN; + +import com.google.common.annotations.VisibleForTesting; import java.util.ArrayList; import java.util.List; +import java.util.Optional; +import java.util.PriorityQueue; +import java.util.Set; +import java.util.function.BiPredicate; +import java.util.function.BinaryOperator; +import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; +import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.data.type.WideningTypeRule; import org.opensearch.sql.exception.ExpressionEvaluationException; -public class CoercionUtils { - +public final class CoercionUtils { /** * Casts the arguments to the types specified in the typeChecker. Returns null if no combination * of parameter types matches the arguments or if casting fails. @@ -31,15 +39,26 @@ public class CoercionUtils { RexBuilder builder, PPLTypeChecker typeChecker, List arguments) { List> paramTypeCombinations = typeChecker.getParameterTypes(); - // TODO: var args? - + List sourceTypes = + arguments.stream() + .map(node -> OpenSearchTypeFactory.convertRelDataTypeToExprType(node.getType())) + .collect(Collectors.toList()); + // Candidate parameter signatures ordered by decreasing widening distance + PriorityQueue, Integer>> rankedSignatures = + new PriorityQueue<>((left, right) -> Integer.compare(right.getValue(), left.getValue())); for (List paramTypes : paramTypeCombinations) { - List castedArguments = castArguments(builder, paramTypes, arguments); - if (castedArguments != null) { - return castedArguments; + int distance = distance(sourceTypes, paramTypes); + if (distance == TYPE_EQUAL) { + return castArguments(builder, paramTypes, arguments); } + Optional.of(distance) + .filter(value -> value != IMPOSSIBLE_WIDENING) + .ifPresent(value -> rankedSignatures.add(Pair.of(paramTypes, value))); } - return null; + return Optional.ofNullable(rankedSignatures.peek()) + .map(Pair::getKey) + .map(paramTypes -> castArguments(builder, paramTypes, arguments)) + .orElse(null); } /** @@ -90,11 +109,16 @@ public class CoercionUtils { if (!argType.shouldCast(targetType)) { return arg; } - - if (WideningTypeRule.distance(argType, targetType) != WideningTypeRule.IMPOSSIBLE_WIDENING) { - return builder.makeCast(OpenSearchTypeFactory.convertExprTypeToRelDataType(targetType), arg); + if (distance(argType, targetType) != IMPOSSIBLE_WIDENING) { + return builder.makeCast( + OpenSearchTypeFactory.convertExprTypeToRelDataType(targetType), arg, true, true); } - return null; + return resolveCommonType(argType, targetType) + .map( + exprType -> + builder.makeCast( + OpenSearchTypeFactory.convertExprTypeToRelDataType(exprType), arg, true, true)) + .orElse(null); } /** @@ -118,12 +142,8 @@ public class CoercionUtils { for (int i = 1; i < arguments.size(); i++) { var type = OpenSearchTypeFactory.convertRelDataTypeToExprType(arguments.get(i).getType()); try { - if (areDateAndTime(widestType, type)) { - // If one is date and the other is time, we consider timestamp as the widest type - widestType = ExprCoreType.TIMESTAMP; - } else { - widestType = WideningTypeRule.max(widestType, type); - } + final ExprType tempType = widestType; + widestType = resolveCommonType(widestType, type).orElseGet(() -> max(tempType, type)); } catch (ExpressionEvaluationException e) { // the two types are not compatible, return null return null; @@ -136,4 +156,119 @@ private static boolean areDateAndTime(ExprType type1, ExprType type2) { return (type1 == ExprCoreType.DATE && type2 == ExprCoreType.TIME) || (type1 == ExprCoreType.TIME && type2 == ExprCoreType.DATE); } + + @VisibleForTesting + public static Optional resolveCommonType(ExprType left, ExprType right) { + return COMMON_COERCION_RULES.stream() + .map(rule -> rule.apply(left, right)) + .flatMap(Optional::stream) + .findFirst(); + } + + public static boolean hasString(List rexNodeList) { + return rexNodeList.stream() + .map(RexNode::getType) + .map(OpenSearchTypeFactory::convertRelDataTypeToExprType) + .anyMatch(t -> t == ExprCoreType.STRING); + } + + private static final Set NUMBER_TYPES = ExprCoreType.numberTypes(); + + private static final List COMMON_COERCION_RULES = + List.of( + CoercionRule.of( + (left, right) -> areDateAndTime(left, right), + (left, right) -> ExprCoreType.TIMESTAMP), + CoercionRule.of( + (left, right) -> hasString(left, right) && hasNumber(left, right), + (left, right) -> ExprCoreType.DOUBLE)); + + private static boolean hasString(ExprType left, ExprType right) { + return left == ExprCoreType.STRING || right == ExprCoreType.STRING; + } + + private static boolean hasNumber(ExprType left, ExprType right) { + return NUMBER_TYPES.contains(left) || NUMBER_TYPES.contains(right); + } + + private static boolean hasBoolean(ExprType left, ExprType right) { + return left == ExprCoreType.BOOLEAN || right == ExprCoreType.BOOLEAN; + } + + private record CoercionRule( + BiPredicate predicate, BinaryOperator resolver) { + + Optional apply(ExprType left, ExprType right) { + return predicate.test(left, right) + ? Optional.of(resolver.apply(left, right)) + : Optional.empty(); + } + + static CoercionRule of( + BiPredicate predicate, BinaryOperator resolver) { + return new CoercionRule(predicate, resolver); + } + } + + private static final int IMPOSSIBLE_WIDENING = Integer.MAX_VALUE; + private static final int TYPE_EQUAL = 0; + + private static int distance(ExprType type1, ExprType type2) { + return distance(type1, type2, TYPE_EQUAL); + } + + private static int distance(ExprType type1, ExprType type2, int distance) { + if (type1 == type2) { + return distance; + } else if (type1 == UNKNOWN) { + return IMPOSSIBLE_WIDENING; + } else if (type1 == ExprCoreType.STRING && type2 == ExprCoreType.DOUBLE) { + return 1; + } else { + return type1.getParent().stream() + .map(parentOfType1 -> distance(parentOfType1, type2, distance + 1)) + .reduce(Math::min) + .get(); + } + } + + /** + * The max type among two types. The max is defined as follow if type1 could widen to type2, then + * max is type2, vice versa if type1 couldn't widen to type2 and type2 could't widen to type1, + * then throw {@link ExpressionEvaluationException}. + * + * @param type1 type1 + * @param type2 type2 + * @return the max type among two types. + */ + public static ExprType max(ExprType type1, ExprType type2) { + int type1To2 = distance(type1, type2); + int type2To1 = distance(type2, type1); + + if (type1To2 == Integer.MAX_VALUE && type2To1 == Integer.MAX_VALUE) { + throw new ExpressionEvaluationException( + String.format("no max type of %s and %s ", type1, type2)); + } else { + return type1To2 == Integer.MAX_VALUE ? type1 : type2; + } + } + + public static int distance(List sourceTypes, List targetTypes) { + if (sourceTypes.size() != targetTypes.size()) { + return IMPOSSIBLE_WIDENING; + } + + int totalDistance = 0; + for (int i = 0; i < sourceTypes.size(); i++) { + ExprType source = sourceTypes.get(i); + ExprType target = targetTypes.get(i); + int distance = distance(source, target); + if (distance == IMPOSSIBLE_WIDENING) { + return IMPOSSIBLE_WIDENING; + } else { + totalDistance += distance; + } + } + return totalDistance; + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendCore.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendCore.java new file mode 100644 index 00000000000..f9a67e4d6d8 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendCore.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.CollectionUDF; + +import java.util.ArrayList; +import java.util.List; + +/** Core logic for `mvappend` command to collect elements from list of args */ +public class MVAppendCore { + + /** + * Collect non-null elements from `args`. If an item is a list, it will collect non-null elements + * of the list. See {@ref MVAppendFunctionImplTest} for detailed behavior. + */ + public static List collectElements(Object... args) { + List elements = new ArrayList<>(); + + for (Object arg : args) { + if (arg == null) { + continue; + } else if (arg instanceof List) { + addListElements((List) arg, elements); + } else { + elements.add(arg); + } + } + + return elements.isEmpty() ? null : elements; + } + + private static void addListElements(List list, List elements) { + for (Object item : list) { + if (item != null) { + elements.add(item); + } + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java new file mode 100644 index 00000000000..107df5eea4e --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java @@ -0,0 +1,102 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.CollectionUDF; + +import static org.apache.calcite.sql.type.SqlTypeUtil.createArrayType; + +import java.util.List; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.SqlOperatorBinding; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +/** + * MVAppend function that appends all elements from arguments to create an array. Always returns an + * array or null for consistent type behavior. + */ +public class MVAppendFunctionImpl extends ImplementorUDF { + + public MVAppendFunctionImpl() { + super(new MVAppendImplementor(), NullPolicy.ALL); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return sqlOperatorBinding -> { + RelDataTypeFactory typeFactory = sqlOperatorBinding.getTypeFactory(); + + if (sqlOperatorBinding.getOperandCount() == 0) { + return typeFactory.createSqlType(SqlTypeName.NULL); + } + + RelDataType elementType = determineElementType(sqlOperatorBinding, typeFactory); + return createArrayType( + typeFactory, typeFactory.createTypeWithNullability(elementType, true), true); + }; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + + private static RelDataType determineElementType( + SqlOperatorBinding sqlOperatorBinding, RelDataTypeFactory typeFactory) { + RelDataType mostGeneralType = null; + + for (int i = 0; i < sqlOperatorBinding.getOperandCount(); i++) { + RelDataType operandType = getComponentType(sqlOperatorBinding.getOperandType(i)); + + mostGeneralType = updateMostGeneralType(mostGeneralType, operandType, typeFactory); + } + + return mostGeneralType != null ? mostGeneralType : typeFactory.createSqlType(SqlTypeName.NULL); + } + + private static RelDataType getComponentType(RelDataType operandType) { + if (!operandType.isStruct() && operandType.getComponentType() != null) { + return operandType.getComponentType(); + } + return operandType; + } + + private static RelDataType updateMostGeneralType( + RelDataType current, RelDataType candidate, RelDataTypeFactory typeFactory) { + if (current == null) { + return candidate; + } + + if (!current.equals(candidate)) { + return typeFactory.createSqlType(SqlTypeName.ANY); + } else { + return current; + } + } + + public static class MVAppendImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + return Expressions.call( + Types.lookupMethod(MVAppendFunctionImpl.class, "mvappend", Object[].class), + Expressions.newArrayInit(Object.class, translatedOperands)); + } + } + + public static Object mvappend(Object... args) { + return MVAppendCore.collectElements(args); + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MapAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MapAppendFunctionImpl.java new file mode 100644 index 00000000000..4cb0acae612 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MapAppendFunctionImpl.java @@ -0,0 +1,115 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.CollectionUDF; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +/** + * MapAppend function that merges two maps. All the values will be converted to list for type + * consistency. + */ +public class MapAppendFunctionImpl extends ImplementorUDF { + + public MapAppendFunctionImpl() { + super(new MapAppendImplementor(), NullPolicy.ALL); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return sqlOperatorBinding -> { + RelDataTypeFactory typeFactory = sqlOperatorBinding.getTypeFactory(); + return typeFactory.createMapType( + typeFactory.createSqlType(SqlTypeName.VARCHAR), + typeFactory.createSqlType(SqlTypeName.ANY)); + }; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + + public static class MapAppendImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + if (translatedOperands.size() != 2) { + throw new IllegalArgumentException("MAP_APPEND function requires exactly 2 arguments"); + } + + return Expressions.call( + Types.lookupMethod(MapAppendFunctionImpl.class, "mapAppend", Object.class, Object.class), + translatedOperands.get(0), + translatedOperands.get(1)); + } + } + + public static Object mapAppend(Object map1, Object map2) { + if (map1 == null && map2 == null) { + return null; + } + if (map1 == null) { + return mapAppendImpl(verifyMap(map2)); + } + if (map2 == null) { + return mapAppendImpl(verifyMap(map1)); + } + + return mapAppendImpl(verifyMap(map1), verifyMap(map2)); + } + + @SuppressWarnings("unchecked") + private static Map verifyMap(Object map) { + if (!(map instanceof Map)) { + throw new IllegalArgumentException( + "MAP_APPEND function requires both arguments to be MAP type"); + } + return (Map) map; + } + + static Map mapAppendImpl(Map map) { + Map result = new HashMap<>(); + for (String key : map.keySet()) { + result.put(key, MVAppendCore.collectElements(map.get(key))); + } + return result; + } + + static Map mapAppendImpl( + Map firstMap, Map secondMap) { + Map result = new HashMap<>(); + + for (String key : mergeKeys(firstMap, secondMap)) { + result.put(key, MVAppendCore.collectElements(firstMap.get(key), secondMap.get(key))); + } + + return result; + } + + private static Set mergeKeys( + Map firstMap, Map secondMap) { + Set keys = new HashSet<>(); + keys.addAll(firstMap.keySet()); + keys.addAll(secondMap.keySet()); + return keys; + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MapRemoveFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MapRemoveFunctionImpl.java new file mode 100644 index 00000000000..1f86fcbe636 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MapRemoveFunctionImpl.java @@ -0,0 +1,101 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.CollectionUDF; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +/** + * Internal MAP_REMOVE function that removes specified keys from a map. Function signature: + * map_remove(map, array_of_keys) -> map Used internally for dynamic fields implementation to dedupe + * field names in _MAP. + */ +public class MapRemoveFunctionImpl extends ImplementorUDF { + + public MapRemoveFunctionImpl() { + super(new MapRemoveImplementor(), NullPolicy.ARG0); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return sqlOperatorBinding -> { + // Return type is the same as the first argument (the map) + RelDataType mapType = sqlOperatorBinding.getOperandType(0); + return sqlOperatorBinding.getTypeFactory().createTypeWithNullability(mapType, true); + }; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + + public static class MapRemoveImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + return Expressions.call( + Types.lookupMethod(MapRemoveFunctionImpl.class, "mapRemove", Object.class, Object.class), + translatedOperands.get(0), + translatedOperands.get(1)); + } + } + + /** + * Removes specified keys from a map. + * + * @param mapArg the input map + * @param keysArg the array/list of keys to remove + * @return a new map with the specified keys removed, or null if input map is null + */ + @SuppressWarnings("unchecked") + public static Object mapRemove(Object mapArg, Object keysArg) { + if (mapArg == null || keysArg == null) { + return mapArg; + } + + verifyArgTypes(mapArg, keysArg); + + return mapRemove((Map) mapArg, (List) keysArg); + } + + private static void verifyArgTypes(Object mapArg, Object keysArg) { + if (!(mapArg instanceof Map)) { + throw new IllegalArgumentException("First argument must be a map, got: " + mapArg.getClass()); + } + + if (!(keysArg instanceof List)) { + throw new IllegalArgumentException( + "Second argument must be an array/list, got: " + keysArg.getClass()); + } + } + + private static Map mapRemove( + Map originalMap, List keysToRemove) { + Map resultMap = new HashMap<>(originalMap); + + for (Object keyObj : keysToRemove) { + if (keyObj != null) { + String key = keyObj.toString(); + resultMap.remove(key); + } + } + + return resultMap; + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 1097a9a5b0d..68eb0ed5cca 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -46,12 +46,16 @@ import org.opensearch.sql.expression.function.CollectionUDF.ExistsFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.FilterFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.ForallFunctionImpl; +import org.opensearch.sql.expression.function.CollectionUDF.MVAppendFunctionImpl; +import org.opensearch.sql.expression.function.CollectionUDF.MapAppendFunctionImpl; +import org.opensearch.sql.expression.function.CollectionUDF.MapRemoveFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.ReduceFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.TransformFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonAppendFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonArrayLengthFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonDeleteFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonExtendFunctionImpl; +import org.opensearch.sql.expression.function.jsonUDF.JsonExtractAllFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonExtractFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonKeysFunctionImpl; @@ -90,7 +94,6 @@ import org.opensearch.sql.expression.function.udf.ip.CidrMatchFunction; import org.opensearch.sql.expression.function.udf.ip.CompareIpFunction; import org.opensearch.sql.expression.function.udf.ip.IPFunction; -import org.opensearch.sql.expression.function.udf.math.CRC32Function; import org.opensearch.sql.expression.function.udf.math.ConvFunction; import org.opensearch.sql.expression.function.udf.math.DivideFunction; import org.opensearch.sql.expression.function.udf.math.EulerFunction; @@ -111,6 +114,8 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { new JsonArrayLengthFunctionImpl().toUDF("JSON_ARRAY_LENGTH"); public static final SqlOperator JSON_EXTRACT = new JsonExtractFunctionImpl().toUDF("JSON_EXTRACT"); + public static final SqlOperator JSON_EXTRACT_ALL = + new JsonExtractAllFunctionImpl().toUDF("JSON_EXTRACT_ALL"); public static final SqlOperator JSON_KEYS = new JsonKeysFunctionImpl().toUDF("JSON_KEYS"); public static final SqlOperator JSON_SET = new JsonSetFunctionImpl().toUDF("JSON_SET"); public static final SqlOperator JSON_DELETE = new JsonDeleteFunctionImpl().toUDF("JSON_DELETE"); @@ -122,7 +127,6 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { public static final SqlOperator E = new EulerFunction().toUDF("E"); public static final SqlOperator CONV = new ConvFunction().toUDF("CONVERT"); public static final SqlOperator MOD = new ModFunction().toUDF("MOD"); - public static final SqlOperator CRC32 = new CRC32Function().toUDF("CRC32"); public static final SqlOperator DIVIDE = new DivideFunction().toUDF("DIVIDE"); public static final SqlOperator SHA2 = CryptographicFunction.sha2().toUDF("SHA2"); public static final SqlOperator CIDRMATCH = new CidrMatchFunction().toUDF("CIDRMATCH"); @@ -383,6 +387,9 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { public static final SqlOperator FORALL = new ForallFunctionImpl().toUDF("forall"); public static final SqlOperator EXISTS = new ExistsFunctionImpl().toUDF("exists"); public static final SqlOperator ARRAY = new ArrayFunctionImpl().toUDF("array"); + public static final SqlOperator MAP_APPEND = new MapAppendFunctionImpl().toUDF("map_append"); + public static final SqlOperator MAP_REMOVE = new MapRemoveFunctionImpl().toUDF("MAP_REMOVE"); + public static final SqlOperator MVAPPEND = new MVAppendFunctionImpl().toUDF("mvappend"); public static final SqlOperator FILTER = new FilterFunctionImpl().toUDF("filter"); public static final SqlOperator TRANSFORM = new TransformFunctionImpl().toUDF("transform"); public static final SqlOperator REDUCE = new ReduceFunctionImpl().toUDF("reduce"); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index eeeed029b69..c85a429a81d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -99,6 +99,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_DELETE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_EXTEND; import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_EXTRACT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_EXTRACT_ALL; import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_KEYS; import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_OBJECT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_SET; @@ -123,6 +124,9 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.LTRIM; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAKEDATE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAKETIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAP_APPEND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAP_CONCAT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAP_REMOVE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MATCH; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MATCH_BOOL_PREFIX; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MATCH_PHRASE; @@ -144,6 +148,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLY; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLYFUNCTION; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTI_MATCH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MVAPPEND; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MVJOIN; import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOTEQUAL; @@ -238,12 +243,15 @@ import java.util.Optional; import java.util.StringJoiner; import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nullable; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexLambda; +import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlOperator; @@ -251,6 +259,7 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.fun.SqlTrimFunction.Flag; import org.apache.calcite.sql.type.CompositeOperandTypeChecker; +import org.apache.calcite.sql.type.FamilyOperandTypeChecker; import org.apache.calcite.sql.type.ImplicitCastOperandTypeChecker; import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SameOperandTypeChecker; @@ -409,10 +418,13 @@ public void registerExternalAggOperator( aggExternalFunctionRegistry.put(functionName, Pair.of(signature, handler)); } - public void validateAggFunctionSignature( - BuiltinFunctionName functionName, RexNode field, List argList) { + public List validateAggFunctionSignature( + BuiltinFunctionName functionName, + RexNode field, + List argList, + RexBuilder rexBuilder) { var implementation = getImplementation(functionName); - validateFunctionArgs(implementation, functionName, field, argList); + return validateFunctionArgs(implementation, functionName, field, argList, rexBuilder); } public RelBuilder.AggCall resolveAgg( @@ -424,17 +436,21 @@ public RelBuilder.AggCall resolveAgg( var implementation = getImplementation(functionName); // Validation is done based on original argument types to generate error from user perspective. - validateFunctionArgs(implementation, functionName, field, argList); + List nodes = + validateFunctionArgs(implementation, functionName, field, argList, context.rexBuilder); var handler = implementation.getValue(); - return handler.apply(distinct, field, argList, context); + return nodes != null + ? handler.apply(distinct, nodes.getFirst(), nodes.subList(1, nodes.size()), context) + : handler.apply(distinct, field, argList, context); } - static void validateFunctionArgs( + static List validateFunctionArgs( Pair implementation, BuiltinFunctionName functionName, RexNode field, - List argList) { + List argList, + RexBuilder rexBuilder) { CalciteFuncSignature signature = implementation.getKey(); List argTypes = new ArrayList<>(); @@ -447,19 +463,29 @@ static void validateFunctionArgs( List additionalArgTypes = argList.stream().map(PlanUtils::derefMapCall).map(RexNode::getType).toList(); argTypes.addAll(additionalArgTypes); + List coercionNodes = null; if (!signature.match(functionName.getName(), argTypes)) { - String errorMessagePattern = - argTypes.size() <= 1 - ? "Aggregation function %s expects field type {%s}, but got %s" - : "Aggregation function %s expects field type and additional arguments {%s}, but got" - + " %s"; - throw new ExpressionEvaluationException( - String.format( - errorMessagePattern, - functionName, - signature.typeChecker().getAllowedSignatures(), - PlanUtils.getActualSignature(argTypes))); + List fields = new ArrayList<>(); + fields.add(field); + fields.addAll(argList); + if (CoercionUtils.hasString(fields)) { + coercionNodes = CoercionUtils.castArguments(rexBuilder, signature.typeChecker(), fields); + } + if (coercionNodes == null) { + String errorMessagePattern = + argTypes.size() <= 1 + ? "Aggregation function %s expects field type {%s}, but got %s" + : "Aggregation function %s expects field type and additional arguments {%s}, but" + + " got %s"; + throw new ExpressionEvaluationException( + String.format( + errorMessagePattern, + functionName, + signature.typeChecker().getAllowedSignatures(), + PlanUtils.getActualSignature(argTypes))); + } } + return coercionNodes; } private Pair getImplementation( @@ -642,6 +668,20 @@ protected void registerOperator( typeChecker); } + protected void registerDivideFunction(BuiltinFunctionName functionName) { + register( + functionName, + (FunctionImp2) + (builder, left, right) -> { + SqlOperator operator = + CalcitePlanContext.isLegacyPreferred() + ? PPLBuiltinOperators.DIVIDE + : SqlLibraryOperators.SAFE_DIVIDE; + return builder.makeCall(operator, left, right); + }, + PPLTypeChecker.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC)); + } + void populate() { // register operators for comparison registerOperator(NOTEQUAL, PPLBuiltinOperators.NOT_EQUALS_IP, SqlStdOperatorTable.NOT_EQUALS); @@ -658,8 +698,14 @@ void populate() { // Register ADDFUNCTION for numeric addition only registerOperator(ADDFUNCTION, SqlStdOperatorTable.PLUS); - registerOperator(SUBTRACT, SqlStdOperatorTable.MINUS); - registerOperator(SUBTRACTFUNCTION, SqlStdOperatorTable.MINUS); + registerOperator( + SUBTRACTFUNCTION, + SqlStdOperatorTable.MINUS, + PPLTypeChecker.wrapFamily((FamilyOperandTypeChecker) OperandTypes.NUMERIC_NUMERIC)); + registerOperator( + SUBTRACT, + SqlStdOperatorTable.MINUS, + PPLTypeChecker.wrapFamily((FamilyOperandTypeChecker) OperandTypes.NUMERIC_NUMERIC)); registerOperator(MULTIPLY, SqlStdOperatorTable.MULTIPLY); registerOperator(MULTIPLYFUNCTION, SqlStdOperatorTable.MULTIPLY); registerOperator(TRUNCATE, SqlStdOperatorTable.TRUNCATE); @@ -668,20 +714,86 @@ void populate() { registerOperator(LOWER, SqlStdOperatorTable.LOWER); registerOperator(POSITION, SqlStdOperatorTable.POSITION); registerOperator(LOCATE, SqlStdOperatorTable.POSITION); - registerOperator(REPLACE, SqlStdOperatorTable.REPLACE); + // Register REPLACE with automatic PCRE-to-Java backreference conversion + register( + REPLACE, + (RexBuilder builder, RexNode... args) -> { + // Validate regex pattern at query planning time + if (args.length >= 2 && args[1] instanceof RexLiteral) { + RexLiteral patternLiteral = (RexLiteral) args[1]; + String pattern = patternLiteral.getValueAs(String.class); + if (pattern != null) { + try { + // Compile pattern to validate it - this will throw PatternSyntaxException if + // invalid + Pattern.compile(pattern); + } catch (PatternSyntaxException e) { + // Convert to IllegalArgumentException so it's treated as a client error (400) + throw new IllegalArgumentException( + String.format("Invalid regex pattern '%s': %s", pattern, e.getDescription()), + e); + } + } + } + + if (args.length == 3 && args[2] instanceof RexLiteral) { + RexLiteral literal = (RexLiteral) args[2]; + String replacement = literal.getValueAs(String.class); + if (replacement != null) { + // Convert PCRE/sed backreferences (\1, \2) to Java style ($1, $2) + String javaReplacement = replacement.replaceAll("\\\\(\\d+)", "\\$$1"); + if (!javaReplacement.equals(replacement)) { + RexNode convertedLiteral = + builder.makeLiteral( + javaReplacement, + literal.getType(), + literal.getTypeName() != SqlTypeName.CHAR); + return builder.makeCall( + SqlLibraryOperators.REGEXP_REPLACE_3, args[0], args[1], convertedLiteral); + } + } + } + return builder.makeCall(SqlLibraryOperators.REGEXP_REPLACE_3, args); + }, + wrapSqlOperandTypeChecker( + SqlLibraryOperators.REGEXP_REPLACE_3.getOperandTypeChecker(), REPLACE.name(), false)); registerOperator(UPPER, SqlStdOperatorTable.UPPER); registerOperator(ABS, SqlStdOperatorTable.ABS); registerOperator(ACOS, SqlStdOperatorTable.ACOS); registerOperator(ASIN, SqlStdOperatorTable.ASIN); registerOperator(ATAN, SqlStdOperatorTable.ATAN); registerOperator(ATAN2, SqlStdOperatorTable.ATAN2); - registerOperator(CEIL, SqlStdOperatorTable.CEIL); - registerOperator(CEILING, SqlStdOperatorTable.CEIL); + // TODO, workaround to support sequence CompositeOperandTypeChecker. + registerOperator( + CEIL, + SqlStdOperatorTable.CEIL, + PPLTypeChecker.wrapComposite( + (CompositeOperandTypeChecker) + OperandTypes.NUMERIC_OR_INTERVAL.or( + OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.ANY)), + false)); + // TODO, workaround to support sequence CompositeOperandTypeChecker. + registerOperator( + CEILING, + SqlStdOperatorTable.CEIL, + PPLTypeChecker.wrapComposite( + (CompositeOperandTypeChecker) + OperandTypes.NUMERIC_OR_INTERVAL.or( + OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.ANY)), + false)); registerOperator(COS, SqlStdOperatorTable.COS); registerOperator(COT, SqlStdOperatorTable.COT); registerOperator(DEGREES, SqlStdOperatorTable.DEGREES); registerOperator(EXP, SqlStdOperatorTable.EXP); - registerOperator(FLOOR, SqlStdOperatorTable.FLOOR); + // TODO, workaround to support sequence CompositeOperandTypeChecker. + registerOperator( + FLOOR, + SqlStdOperatorTable.FLOOR, + PPLTypeChecker.wrapComposite( + (CompositeOperandTypeChecker) + OperandTypes.NUMERIC_OR_INTERVAL.or( + OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.ANY)), + false)); registerOperator(LN, SqlStdOperatorTable.LN); registerOperator(LOG10, SqlStdOperatorTable.LOG10); registerOperator(PI, SqlStdOperatorTable.PI); @@ -689,7 +801,15 @@ void populate() { registerOperator(POWER, SqlStdOperatorTable.POWER); registerOperator(RADIANS, SqlStdOperatorTable.RADIANS); registerOperator(RAND, SqlStdOperatorTable.RAND); - registerOperator(ROUND, SqlStdOperatorTable.ROUND); + // TODO, workaround to support sequence CompositeOperandTypeChecker. + registerOperator( + ROUND, + SqlStdOperatorTable.ROUND, + PPLTypeChecker.wrapComposite( + (CompositeOperandTypeChecker) + OperandTypes.NUMERIC.or( + OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.INTEGER)), + false)); registerOperator(SIGN, SqlStdOperatorTable.SIGN); registerOperator(SIGNUM, SqlStdOperatorTable.SIGN); registerOperator(SIN, SqlStdOperatorTable.SIN); @@ -712,6 +832,7 @@ void populate() { registerOperator(LOG2, SqlLibraryOperators.LOG2); registerOperator(MD5, SqlLibraryOperators.MD5); registerOperator(SHA1, SqlLibraryOperators.SHA1); + registerOperator(CRC32, SqlLibraryOperators.CRC32); registerOperator(INTERNAL_REGEXP_REPLACE_3, SqlLibraryOperators.REGEXP_REPLACE_3); registerOperator(INTERNAL_REGEXP_REPLACE_PG_4, SqlLibraryOperators.REGEXP_REPLACE_PG_4); registerOperator(INTERNAL_REGEXP_REPLACE_5, SqlLibraryOperators.REGEXP_REPLACE_5); @@ -732,9 +853,8 @@ void populate() { registerOperator(MOD, PPLBuiltinOperators.MOD); registerOperator(MODULUS, PPLBuiltinOperators.MOD); registerOperator(MODULUSFUNCTION, PPLBuiltinOperators.MOD); - registerOperator(CRC32, PPLBuiltinOperators.CRC32); - registerOperator(DIVIDE, PPLBuiltinOperators.DIVIDE); - registerOperator(DIVIDEFUNCTION, PPLBuiltinOperators.DIVIDE); + registerDivideFunction(DIVIDE); + registerDivideFunction(DIVIDEFUNCTION); registerOperator(SHA2, PPLBuiltinOperators.SHA2); registerOperator(CIDRMATCH, PPLBuiltinOperators.CIDRMATCH); registerOperator(INTERNAL_GROK, PPLBuiltinOperators.GROK); @@ -834,6 +954,10 @@ void populate() { PPLTypeChecker.family(SqlTypeFamily.ARRAY, SqlTypeFamily.CHARACTER)); registerOperator(ARRAY, PPLBuiltinOperators.ARRAY); + registerOperator(MVAPPEND, PPLBuiltinOperators.MVAPPEND); + registerOperator(MAP_APPEND, PPLBuiltinOperators.MAP_APPEND); + registerOperator(MAP_CONCAT, SqlLibraryOperators.MAP_CONCAT); + registerOperator(MAP_REMOVE, PPLBuiltinOperators.MAP_REMOVE); registerOperator(ARRAY_LENGTH, SqlLibraryOperators.ARRAY_LENGTH); registerOperator(FORALL, PPLBuiltinOperators.FORALL); registerOperator(EXISTS, PPLBuiltinOperators.EXISTS); @@ -867,6 +991,7 @@ void populate() { registerOperator(JSON_DELETE, PPLBuiltinOperators.JSON_DELETE); registerOperator(JSON_APPEND, PPLBuiltinOperators.JSON_APPEND); registerOperator(JSON_EXTEND, PPLBuiltinOperators.JSON_EXTEND); + registerOperator(JSON_EXTRACT_ALL, PPLBuiltinOperators.JSON_EXTRACT_ALL); // internal // Register operators with a different type checker @@ -898,19 +1023,15 @@ void populate() { XOR, SqlStdOperatorTable.NOT_EQUALS, PPLTypeChecker.family(SqlTypeFamily.BOOLEAN, SqlTypeFamily.BOOLEAN)); - // SqlStdOperatorTable.CASE.getOperandTypeChecker is null. We manually create a - // type checker - // for it. The second and third operands are required to be of the same type. If - // not, - // it will throw an IllegalArgumentException with information Can't find - // leastRestrictive type + // SqlStdOperatorTable.CASE.getOperandTypeChecker is null. We manually create a type checker + // for it. The second and third operands are required to be of the same type. If not, it will + // throw an IllegalArgumentException with information Can't find leastRestrictive type registerOperator( IF, SqlStdOperatorTable.CASE, PPLTypeChecker.family(SqlTypeFamily.BOOLEAN, SqlTypeFamily.ANY, SqlTypeFamily.ANY)); // Re-define the type checker for is not null, is present, and is null since - // their original - // type checker ANY isn't compatible with struct types. + // their original type checker ANY isn't compatible with struct types. registerOperator( IS_NOT_NULL, SqlStdOperatorTable.IS_NOT_NULL, diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractAllFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractAllFunctionImpl.java new file mode 100644 index 00000000000..1f91c87bb77 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractAllFunctionImpl.java @@ -0,0 +1,218 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.TYPE_FACTORY; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Stack; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeFamily; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +/** + * UDF which extract all the fields from JSON to a MAP. Items are collected from input JSON and + * stored with the key of their path in the JSON. This UDF is designed to be used for `spath` + * command without path param. See {@ref JsonExtractAllFunctionImplTest} for the detailed spec. + */ +public class JsonExtractAllFunctionImpl extends ImplementorUDF { + private static final String ARRAY_SUFFIX = "{}"; + private static final JsonFactory JSON_FACTORY = new JsonFactory(); + + public JsonExtractAllFunctionImpl() { + super(new JsonExtractAllImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return ReturnTypes.explicit( + TYPE_FACTORY.createMapType( + TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR), + TYPE_FACTORY.createSqlType(SqlTypeName.ANY), + true)); + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return UDFOperandMetadata.wrap(OperandTypes.family(SqlTypeFamily.STRING)); + } + + public static class JsonExtractAllImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonExtractAllFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); + } + } + + public static Object eval(Object... args) { + if (args.length < 1) { + return null; + } + + String jsonStr = (String) args[0]; + if (jsonStr == null || jsonStr.trim().isEmpty()) { + return null; + } + + return parseJson(jsonStr); + } + + private static Map parseJson(String jsonStr) { + Map resultMap = new HashMap<>(); + Stack pathStack = new Stack<>(); + + try (JsonParser parser = JSON_FACTORY.createParser(jsonStr)) { + JsonToken token; + + while ((token = parser.nextToken()) != null) { + switch (token) { + case START_OBJECT: + break; + + case END_OBJECT: + if (!pathStack.isEmpty() && !isInArray(pathStack)) { + pathStack.pop(); + } + break; + + case START_ARRAY: + pathStack.push(ARRAY_SUFFIX); + break; + + case END_ARRAY: + pathStack.pop(); + if (!pathStack.isEmpty() && !isInArray(pathStack)) { + pathStack.pop(); + } + break; + + case FIELD_NAME: + String fieldName = parser.currentName(); + pathStack.push(fieldName); + break; + + case VALUE_STRING: + case VALUE_NUMBER_INT: + case VALUE_NUMBER_FLOAT: + case VALUE_TRUE: + case VALUE_FALSE: + case VALUE_NULL: + if (pathStack.isEmpty()) { + // ignore top level value + return null; + } + + appendValue(resultMap, buildPath(pathStack), extractValue(parser, token)); + + if (!isInArray(pathStack)) { + pathStack.pop(); + } + break; + default: + // Skip other tokens + break; + } + } + } catch (IOException e) { + // ignore exception, and current result will be returned + } + return resultMap; + } + + @SuppressWarnings("unchecked") + private static void appendValue(Map resultMap, String path, Object value) { + Object existingValue = resultMap.get(path); + if (existingValue == null) { + resultMap.put(path, value); + } else if (existingValue instanceof List) { + ((List) existingValue).add(value); + } else { + resultMap.put(path, list(existingValue, value)); + } + } + + private static List list(Object... args) { + List result = new LinkedList<>(); + for (Object arg : args) { + result.add(arg); + } + return result; + } + + private static boolean isInArray(List path) { + return path.size() >= 1 && path.getLast().equals(ARRAY_SUFFIX); + } + + private static Object extractValue(JsonParser parser, JsonToken token) throws IOException { + switch (token) { + case VALUE_STRING: + return parser.getValueAsString(); + case VALUE_NUMBER_INT: + return getIntValue(parser); + case VALUE_NUMBER_FLOAT: + return parser.getDoubleValue(); + case VALUE_TRUE: + return true; + case VALUE_FALSE: + return false; + case VALUE_NULL: + return null; + default: + return parser.getValueAsString(); + } + } + + private static Object getIntValue(JsonParser parser) throws IOException { + if (parser.getNumberType() == JsonParser.NumberType.INT) { + return parser.getIntValue(); + } else if (parser.getNumberType() == JsonParser.NumberType.LONG) { + return parser.getLongValue(); + } else { + // store as double in case of BIG_INTEGER (exceed LONG precision) + return parser.getBigIntegerValue().doubleValue(); + } + } + + private static String buildPath(Collection pathStack) { + StringBuilder builder = new StringBuilder(); + for (String path : pathStack) { + if (ARRAY_SUFFIX.equals(path)) { + builder.append(ARRAY_SUFFIX); + } else if (!path.isEmpty()) { + if (!builder.isEmpty()) { + builder.append("."); + } + builder.append(path); + } + } + return builder.toString(); + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/RexExtractFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/RexExtractFunction.java index fc1a1d0bef6..569e03f84c1 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/RexExtractFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/RexExtractFunction.java @@ -15,11 +15,13 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.CompositeOperandTypeChecker; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; +import org.opensearch.sql.expression.parse.RegexCommonUtils; /** Custom REX_EXTRACT function for extracting regex named capture groups. */ public final class RexExtractFunction extends ImplementorUDF { @@ -35,7 +37,12 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return PPLOperandTypes.STRING_STRING_INTEGER; + // Support both (field, pattern, groupIndex) and (field, pattern, groupName) + return UDFOperandMetadata.wrap( + (CompositeOperandTypeChecker) + PPLOperandTypes.STRING_STRING_INTEGER + .getInnerTypeChecker() + .or(PPLOperandTypes.STRING_STRING_STRING.getInnerTypeChecker())); } private static class RexExtractImplementor implements NotNullImplementor { @@ -45,19 +52,80 @@ public Expression implement( RexToLixTranslator translator, RexCall call, List translatedOperands) { Expression field = translatedOperands.get(0); Expression pattern = translatedOperands.get(1); - Expression groupIndex = translatedOperands.get(2); + Expression groupIndexOrName = translatedOperands.get(2); - return Expressions.call(RexExtractFunction.class, "extractGroup", field, pattern, groupIndex); + return Expressions.call( + RexExtractFunction.class, "extractGroup", field, pattern, groupIndexOrName); } } + /** + * Extract a regex group by index (1-based). + * + * @param text The input text to extract from + * @param pattern The regex pattern + * @param groupIndex The 1-based group index to extract + * @return The extracted value or null if not found or invalid + */ public static String extractGroup(String text, String pattern, int groupIndex) { + if (text == null || pattern == null) { + return null; + } + + return executeExtraction( + text, + pattern, + matcher -> { + if (groupIndex > 0 && groupIndex <= matcher.groupCount()) { + return matcher.group(groupIndex); + } + return null; + }); + } + + /** + * Extract a named capture group from text using the provided pattern. This method avoids the + * index shifting issue that occurs with nested unnamed groups. + * + * @param text The input text to extract from + * @param pattern The regex pattern with named capture groups + * @param groupName The name of the capture group to extract + * @return The extracted value or null if not found + */ + public static String extractGroup(String text, String pattern, String groupName) { + if (text == null || pattern == null || groupName == null) { + return null; + } + + return executeExtraction( + text, + pattern, + matcher -> { + try { + return matcher.group(groupName); + } catch (IllegalArgumentException e) { + // Group name doesn't exist in the pattern + return null; + } + }); + } + + /** + * Common extraction logic to avoid code duplication. + * + * @param text The input text + * @param pattern The regex pattern + * @param extractor Function to extract the value from the matcher + * @return The extracted value or null + */ + private static String executeExtraction( + String text, String pattern, java.util.function.Function extractor) { try { - Pattern compiledPattern = Pattern.compile(pattern); + Pattern compiledPattern = RegexCommonUtils.getCompiledPattern(pattern); Matcher matcher = compiledPattern.matcher(text); - if (matcher.find() && groupIndex > 0 && groupIndex <= matcher.groupCount()) { - return matcher.group(groupIndex); + if (matcher.find()) { + return extractor.apply(matcher); } return null; } catch (PatternSyntaxException e) { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/RexExtractMultiFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/RexExtractMultiFunction.java index 599a518f9ce..2e0cb0cd06c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/RexExtractMultiFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/RexExtractMultiFunction.java @@ -16,11 +16,15 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.CompositeOperandTypeChecker; +import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; +import org.opensearch.sql.expression.parse.RegexCommonUtils; /** Custom REX_EXTRACT_MULTI function for extracting multiple regex matches. */ public final class RexExtractMultiFunction extends ImplementorUDF { @@ -40,7 +44,17 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return PPLOperandTypes.STRING_STRING_INTEGER_INTEGER; + // Support both (field, pattern, groupIndex, maxMatch) and (field, pattern, groupName, maxMatch) + return UDFOperandMetadata.wrap( + (CompositeOperandTypeChecker) + PPLOperandTypes.STRING_STRING_INTEGER_INTEGER + .getInnerTypeChecker() + .or( + OperandTypes.family( + SqlTypeFamily.CHARACTER, + SqlTypeFamily.CHARACTER, + SqlTypeFamily.CHARACTER, + SqlTypeFamily.INTEGER))); } private static class RexExtractMultiImplementor implements NotNullImplementor { @@ -50,7 +64,7 @@ public Expression implement( RexToLixTranslator translator, RexCall call, List translatedOperands) { Expression field = translatedOperands.get(0); Expression pattern = translatedOperands.get(1); - Expression groupIndex = translatedOperands.get(2); + Expression groupIndexOrName = translatedOperands.get(2); Expression maxMatch = translatedOperands.get(3); return Expressions.call( @@ -58,27 +72,97 @@ public Expression implement( "extractMultipleGroups", field, pattern, - groupIndex, + groupIndexOrName, maxMatch); } } + /** + * Extract multiple regex groups by index (1-based). + * + * @param text The input text to extract from + * @param pattern The regex pattern + * @param groupIndex The 1-based group index to extract + * @param maxMatch Maximum number of matches to return (0 = unlimited) + * @return List of extracted values or null if no matches found + */ public static List extractMultipleGroups( String text, String pattern, int groupIndex, int maxMatch) { - // Query planner already validates null inputs via NullPolicy.ARG0 + if (text == null || pattern == null) { + return null; + } + + return executeMultipleExtractions( + text, + pattern, + maxMatch, + matcher -> { + if (groupIndex > 0 && groupIndex <= matcher.groupCount()) { + return matcher.group(groupIndex); + } + return null; + }); + } + + /** + * Extract multiple occurrences of a named capture group from text. This method avoids the index + * shifting issue that occurs with nested unnamed groups. + * + * @param text The input text to extract from + * @param pattern The regex pattern with named capture groups + * @param groupName The name of the capture group to extract + * @param maxMatch Maximum number of matches to return (0 = unlimited) + * @return List of extracted values or null if no matches found + */ + public static List extractMultipleGroups( + String text, String pattern, String groupName, int maxMatch) { + if (text == null || pattern == null || groupName == null) { + return null; + } + + return executeMultipleExtractions( + text, + pattern, + maxMatch, + matcher -> { + try { + return matcher.group(groupName); + } catch (IllegalArgumentException e) { + // Group name doesn't exist in the pattern, stop processing + return null; + } + }); + } + + /** + * Common extraction logic for multiple matches to avoid code duplication. + * + * @param text The input text + * @param pattern The regex pattern + * @param maxMatch Maximum matches (0 = unlimited) + * @param extractor Function to extract the value from the matcher + * @return List of extracted values or null if no matches found + */ + private static List executeMultipleExtractions( + String text, + String pattern, + int maxMatch, + java.util.function.Function extractor) { try { - Pattern compiledPattern = Pattern.compile(pattern); + Pattern compiledPattern = RegexCommonUtils.getCompiledPattern(pattern); Matcher matcher = compiledPattern.matcher(text); List matches = new ArrayList<>(); int matchCount = 0; while (matcher.find() && (maxMatch == 0 || matchCount < maxMatch)) { - if (groupIndex > 0 && groupIndex <= matcher.groupCount()) { - String match = matcher.group(groupIndex); - if (match != null) { - matches.add(match); - matchCount++; - } + String match = extractor.apply(matcher); + if (match != null) { + matches.add(match); + matchCount++; + } else { + // If extractor returns null, it might indicate an error (like invalid group name) + // Stop processing to avoid infinite loop + break; } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/MinspanBucketFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/MinspanBucketFunction.java index 1d8b50687bb..11e1a33afbd 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/MinspanBucketFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/MinspanBucketFunction.java @@ -64,10 +64,10 @@ public Expression implement( return Expressions.call( MinspanBucketImplementor.class, "calculateMinspanBucket", - Expressions.convert_(fieldValue, Number.class), - Expressions.convert_(minSpan, Number.class), - Expressions.convert_(dataRange, Number.class), - Expressions.convert_(maxValue, Number.class)); + Expressions.convert_(Expressions.box(fieldValue), Number.class), + Expressions.convert_(Expressions.box(minSpan), Number.class), + Expressions.convert_(Expressions.box(dataRange), Number.class), + Expressions.convert_(Expressions.box(maxValue), Number.class)); } /** Minspan bucket calculation. */ diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/RangeBucketFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/RangeBucketFunction.java index e838ff04d83..a8f2625b20f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/RangeBucketFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/RangeBucketFunction.java @@ -69,11 +69,11 @@ public Expression implement( return Expressions.call( RangeBucketImplementor.class, "calculateRangeBucket", - Expressions.convert_(fieldValue, Number.class), - Expressions.convert_(dataMin, Number.class), - Expressions.convert_(dataMax, Number.class), - Expressions.convert_(startParam, Number.class), - Expressions.convert_(endParam, Number.class)); + Expressions.convert_(Expressions.box(fieldValue), Number.class), + Expressions.convert_(Expressions.box(dataMin), Number.class), + Expressions.convert_(Expressions.box(dataMax), Number.class), + Expressions.convert_(Expressions.box(startParam), Number.class), + Expressions.convert_(Expressions.box(endParam), Number.class)); } /** Range bucket calculation with expansion algorithm and magnitude-based width. */ diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/SpanBucketFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/SpanBucketFunction.java index 1e6e09f2f26..6970e485525 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/SpanBucketFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/SpanBucketFunction.java @@ -60,8 +60,8 @@ public Expression implement( return Expressions.call( SpanBucketImplementor.class, "calculateSpanBucket", - Expressions.convert_(fieldValue, Number.class), - Expressions.convert_(spanValue, Number.class)); + Expressions.convert_(Expressions.box(fieldValue), Number.class), + Expressions.convert_(Expressions.box(spanValue), Number.class)); } /** Span bucket calculation. */ diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/WidthBucketFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/WidthBucketFunction.java index ef68b17fa14..c1962266248 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/WidthBucketFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/WidthBucketFunction.java @@ -11,9 +11,13 @@ import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; -import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.calcite.type.ExprSqlType; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.binning.BinConstants; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -44,7 +48,20 @@ public WidthBucketFunction() { @Override public SqlReturnTypeInference getReturnTypeInference() { - return ReturnTypes.VARCHAR_2000; + return (opBinding) -> { + RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); + RelDataType arg0Type = opBinding.getOperandType(0); + return dateRelatedType(arg0Type) + ? arg0Type + : typeFactory.createTypeWithNullability( + typeFactory.createSqlType(SqlTypeName.VARCHAR, 2000), true); + }; + } + + public static boolean dateRelatedType(RelDataType type) { + return type instanceof ExprSqlType exprSqlType + && List.of(ExprUDT.EXPR_DATE, ExprUDT.EXPR_TIME, ExprUDT.EXPR_TIMESTAMP) + .contains(exprSqlType.getUdt()); } @Override @@ -65,10 +82,10 @@ public Expression implement( return Expressions.call( WidthBucketImplementor.class, "calculateWidthBucket", - Expressions.convert_(fieldValue, Number.class), - Expressions.convert_(numBins, Number.class), - Expressions.convert_(dataRange, Number.class), - Expressions.convert_(maxValue, Number.class)); + Expressions.convert_(Expressions.box(fieldValue), Number.class), + Expressions.convert_(Expressions.box(numBins), Number.class), + Expressions.convert_(Expressions.box(dataRange), Number.class), + Expressions.convert_(Expressions.box(maxValue), Number.class)); } /** Width bucket calculation using nice number algorithm. */ diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/condition/EnhancedCoalesceFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/condition/EnhancedCoalesceFunction.java index 4ee4f39abd5..719f078ba38 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/condition/EnhancedCoalesceFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/condition/EnhancedCoalesceFunction.java @@ -34,7 +34,7 @@ private static NotNullImplementor createImplementor() { Expressions.call( ExprValueUtils.class, "fromObjectValue", - Expressions.convert_(operand, Object.class))) + Expressions.convert_(Expressions.box(operand), Object.class))) .toList(); Expression returnTypeName = Expressions.constant(call.getType().getSqlTypeName().toString()); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/AddSubDateFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/AddSubDateFunction.java index 8398a508388..9456e6b857a 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/AddSubDateFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/AddSubDateFunction.java @@ -117,13 +117,13 @@ public Expression implement( applyDaysFuncName, properties, base, - Expressions.convert_(temporalDelta, long.class)); + Expressions.convert_(Expressions.box(temporalDelta), long.class)); } else if (SqlTypeFamily.DATETIME_INTERVAL.contains(temporalDeltaType)) { Expression interval = Expressions.call( DateTimeConversionUtils.class, "convertToTemporalAmount", - Expressions.convert_(temporalDelta, long.class), + Expressions.convert_(Expressions.box(temporalDelta), long.class), Expressions.constant( Objects.requireNonNull(temporalDeltaType.getIntervalQualifier()).getUnit())); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/DateAddSubFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/DateAddSubFunction.java index de3a2cb65ab..dc1a4bb1d16 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/DateAddSubFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/DateAddSubFunction.java @@ -66,7 +66,7 @@ public Expression implement( Expressions.call( DateTimeConversionUtils.class, "convertToTemporalAmount", - Expressions.convert_(temporalDelta, long.class), + Expressions.convert_(Expressions.box(temporalDelta), long.class), Expressions.constant( Objects.requireNonNull(temporalDeltaType.getIntervalQualifier()).getUnit())); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/SecToTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/SecToTimeFunction.java index ebeb8019ade..319a22ed372 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/SecToTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/SecToTimeFunction.java @@ -57,7 +57,7 @@ public Expression implement( return Expressions.call( SecToTimeImplementor.class, "secToTime", - Expressions.convert_(translatedOperands.getFirst(), Number.class)); + Expressions.convert_(Expressions.box(translatedOperands.getFirst()), Number.class)); } public static String secToTime(Number seconds) { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/CRC32Function.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/CRC32Function.java deleted file mode 100644 index 98ea55fa349..00000000000 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/CRC32Function.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.expression.function.udf.math; - -import java.util.List; -import java.util.zip.CRC32; -import org.apache.calcite.adapter.enumerable.NotNullImplementor; -import org.apache.calcite.adapter.enumerable.NullPolicy; -import org.apache.calcite.adapter.enumerable.RexToLixTranslator; -import org.apache.calcite.linq4j.tree.Expression; -import org.apache.calcite.linq4j.tree.Expressions; -import org.apache.calcite.rex.RexCall; -import org.apache.calcite.sql.type.ReturnTypes; -import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.opensearch.sql.calcite.utils.PPLOperandTypes; -import org.opensearch.sql.expression.function.ImplementorUDF; -import org.opensearch.sql.expression.function.UDFOperandMetadata; - -/** - * CRC32(value) returns a 32-bit cyclic redundancy check (CRC) checksum - * - *

Signatures: - * - *

    - *
  • (STRING) -> BIGINT - *
- */ -public class CRC32Function extends ImplementorUDF { - public CRC32Function() { - super(new Crc32Implementor(), NullPolicy.ARG0); - } - - @Override - public SqlReturnTypeInference getReturnTypeInference() { - return ReturnTypes.BIGINT_FORCE_NULLABLE; - } - - @Override - public UDFOperandMetadata getOperandMetadata() { - return PPLOperandTypes.STRING; - } - - public static class Crc32Implementor implements NotNullImplementor { - @Override - public Expression implement( - RexToLixTranslator translator, RexCall call, List translatedOperands) { - return Expressions.call(Crc32Implementor.class, "crc32", translatedOperands.getFirst()); - } - - public static long crc32(String value) { - CRC32 crc = new CRC32(); - crc.update(value.getBytes()); - return crc.getValue(); - } - } -} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/DivideFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/DivideFunction.java index d78e75ef31a..1767a2fc69b 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/DivideFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/DivideFunction.java @@ -58,8 +58,8 @@ public Expression implement( return Expressions.call( DivideImplementor.class, "divide", - Expressions.convert_(translatedOperands.get(0), Number.class), - Expressions.convert_(translatedOperands.get(1), Number.class)); + Expressions.convert_(Expressions.box(translatedOperands.get(0)), Number.class), + Expressions.convert_(Expressions.box(translatedOperands.get(1)), Number.class)); } public static Number divide(Number dividend, Number divisor) { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ModFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ModFunction.java index de0cf1f5a81..7a8f8e75f92 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ModFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/math/ModFunction.java @@ -64,14 +64,14 @@ public Expression implement( return Expressions.call( ModImplementor.class, "integralMod", - Expressions.convert_(dividend, Number.class), - Expressions.convert_(divisor, Number.class)); + Expressions.convert_(Expressions.box(dividend), Number.class), + Expressions.convert_(Expressions.box(divisor), Number.class)); } else { return Expressions.call( ModImplementor.class, "floatingMod", - Expressions.convert_(dividend, Number.class), - Expressions.convert_(divisor, Number.class)); + Expressions.convert_(Expressions.box(dividend), Number.class), + Expressions.convert_(Expressions.box(divisor), Number.class)); } } diff --git a/core/src/main/java/org/opensearch/sql/expression/parse/RegexCommonUtils.java b/core/src/main/java/org/opensearch/sql/expression/parse/RegexCommonUtils.java index f2c9092c346..7e194dfbf22 100644 --- a/core/src/main/java/org/opensearch/sql/expression/parse/RegexCommonUtils.java +++ b/core/src/main/java/org/opensearch/sql/expression/parse/RegexCommonUtils.java @@ -22,6 +22,9 @@ public class RegexCommonUtils { private static final Pattern NAMED_GROUP_PATTERN = Pattern.compile("\\(\\?<([a-zA-Z][a-zA-Z0-9]*)>"); + // Pattern to extract ANY named group (valid or invalid) for validation + private static final Pattern ANY_NAMED_GROUP_PATTERN = Pattern.compile("\\(\\?<([^>]+)>"); + private static final int MAX_CACHE_SIZE = 1000; private static final Map patternCache = @@ -50,20 +53,52 @@ public static Pattern getCompiledPattern(String regex) { } /** - * Extract list of named group candidates from a regex pattern. + * Extract list of named group candidates from a regex pattern. Validates that all group names + * conform to Java regex named group requirements: must start with a letter and contain only + * letters and digits. * * @param pattern The regex pattern string - * @return List of named group names found in the pattern + * @return List of valid named group names found in the pattern + * @throws IllegalArgumentException if any named groups contain invalid characters */ public static List getNamedGroupCandidates(String pattern) { ImmutableList.Builder namedGroups = ImmutableList.builder(); - Matcher m = NAMED_GROUP_PATTERN.matcher(pattern); - while (m.find()) { - namedGroups.add(m.group(1)); + + Matcher anyGroupMatcher = ANY_NAMED_GROUP_PATTERN.matcher(pattern); + while (anyGroupMatcher.find()) { + String groupName = anyGroupMatcher.group(1); + + if (!isValidJavaRegexGroupName(groupName)) { + throw new IllegalArgumentException( + String.format( + "Invalid capture group name '%s'. Java regex group names must start with a letter" + + " and contain only letters and digits.", + groupName)); + } } + + Matcher validGroupMatcher = NAMED_GROUP_PATTERN.matcher(pattern); + while (validGroupMatcher.find()) { + namedGroups.add(validGroupMatcher.group(1)); + } + return namedGroups.build(); } + /** + * Validates if a group name conforms to Java regex named group requirements. Java regex group + * names must: - Start with a letter (a-z, A-Z) - Contain only letters (a-z, A-Z) and digits (0-9) + * + * @param groupName The group name to validate + * @return true if valid, false otherwise + */ + private static boolean isValidJavaRegexGroupName(String groupName) { + if (groupName == null || groupName.isEmpty()) { + return false; + } + return groupName.matches("^[A-Za-z][A-Za-z0-9]*$"); + } + /** * Match using find() for partial match semantics with string pattern. * diff --git a/core/src/main/java/org/opensearch/sql/utils/YamlFormatter.java b/core/src/main/java/org/opensearch/sql/utils/YamlFormatter.java index 3ccafb34abd..c50d04f8217 100644 --- a/core/src/main/java/org/opensearch/sql/utils/YamlFormatter.java +++ b/core/src/main/java/org/opensearch/sql/utils/YamlFormatter.java @@ -5,8 +5,10 @@ package org.opensearch.sql.utils; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; @@ -21,11 +23,16 @@ public class YamlFormatter { static { YAMLFactory yamlFactory = new YAMLFactory(); yamlFactory.disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER); + yamlFactory.enable(YAMLGenerator.Feature.USE_PLATFORM_LINE_BREAKS); + yamlFactory.enable(YAMLGenerator.Feature.LITERAL_BLOCK_STYLE); yamlFactory.enable(YAMLGenerator.Feature.MINIMIZE_QUOTES); // Enable smart quoting yamlFactory.enable( YAMLGenerator.Feature.ALWAYS_QUOTE_NUMBERS_AS_STRINGS); // Quote numeric strings yamlFactory.enable(YAMLGenerator.Feature.INDENT_ARRAYS_WITH_INDICATOR); YAML_MAPPER = new ObjectMapper(yamlFactory); + + YAML_MAPPER.setSerializationInclusion(JsonInclude.Include.NON_NULL); + YAML_MAPPER.configure(SerializationFeature.WRITE_NULL_MAP_VALUES, false); } /** diff --git a/core/src/test/java/org/opensearch/sql/analysis/ExpressionReferenceOptimizerTest.java b/core/src/test/java/org/opensearch/sql/analysis/ExpressionReferenceOptimizerTest.java index 28bcb8793fc..091cf532307 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/ExpressionReferenceOptimizerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/ExpressionReferenceOptimizerTest.java @@ -138,6 +138,22 @@ Expression optimize(Expression expression, LogicalPlan logicalPlan) { return optimizer.optimize(DSL.named(expression), new AnalysisContext()); } + @Test + void visitAggregator_with_missing_mapping_returns_original() { + BuiltinFunctionRepository repository = BuiltinFunctionRepository.getInstance(); + LogicalPlan emptyPlan = LogicalPlanDSL.relation("test", table); + ExpressionReferenceOptimizer optimizer = + new ExpressionReferenceOptimizer(repository, emptyPlan); + + Expression unmappedAggregator = DSL.count(DSL.ref("age", INTEGER)); + AnalysisContext context = new AnalysisContext(); + + Expression result = + optimizer.visitAggregator( + (org.opensearch.sql.expression.aggregation.Aggregator) unmappedAggregator, context); + assertEquals(unmappedAggregator, result); + } + LogicalPlan logicalPlan() { return LogicalPlanDSL.aggregation( LogicalPlanDSL.relation("schema", table), diff --git a/core/src/test/java/org/opensearch/sql/analysis/SelectExpressionAnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/SelectExpressionAnalyzerTest.java index a59c875e4ed..4e2cc1faec3 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/SelectExpressionAnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/SelectExpressionAnalyzerTest.java @@ -8,6 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; @@ -84,4 +85,23 @@ protected void assertAnalyzeEqual( NamedExpression expected, UnresolvedExpression unresolvedExpression) { assertEquals(Arrays.asList(expected), analyze(unresolvedExpression)); } + + @Test + public void testContextWrapperIsolation() { + // Test that context wrapper properly isolates optimizer instances, each wrapper should have its + // own optimizer + ExpressionReferenceOptimizer optimizer1 = mock(ExpressionReferenceOptimizer.class); + ExpressionReferenceOptimizer optimizer2 = mock(ExpressionReferenceOptimizer.class); + + AnalysisContext baseContext = new AnalysisContext(); + SelectExpressionAnalyzer.AnalysisContextWithOptimizer wrapper1 = + new SelectExpressionAnalyzer.AnalysisContextWithOptimizer(baseContext, optimizer1); + SelectExpressionAnalyzer.AnalysisContextWithOptimizer wrapper2 = + new SelectExpressionAnalyzer.AnalysisContextWithOptimizer(baseContext, optimizer2); + + assertEquals(baseContext, wrapper1.analysisContext); + assertEquals(baseContext, wrapper2.analysisContext); + assertEquals(optimizer1, wrapper1.optimizer); + assertEquals(optimizer2, wrapper2.optimizer); + } } diff --git a/core/src/test/java/org/opensearch/sql/ast/tree/MultisearchTest.java b/core/src/test/java/org/opensearch/sql/ast/tree/MultisearchTest.java new file mode 100644 index 00000000000..afb26e77089 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/ast/tree/MultisearchTest.java @@ -0,0 +1,105 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.ast.dsl.AstDSL.compare; +import static org.opensearch.sql.ast.dsl.AstDSL.field; +import static org.opensearch.sql.ast.dsl.AstDSL.filter; +import static org.opensearch.sql.ast.dsl.AstDSL.intLiteral; +import static org.opensearch.sql.ast.dsl.AstDSL.relation; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.Node; + +class MultisearchTest { + + @Test + public void testMultisearchCreation() { + // Create subsearches + UnresolvedPlan subsearch1 = relation("table1"); + UnresolvedPlan subsearch2 = + filter(relation("table2"), compare(">", field("age"), intLiteral(30))); + List subsearches = ImmutableList.of(subsearch1, subsearch2); + + // Create multisearch + Multisearch multisearch = new Multisearch(subsearches); + + // Verify properties + assertEquals(subsearches, multisearch.getSubsearches()); + assertEquals(2, multisearch.getSubsearches().size()); + assertNotNull(multisearch.getChild()); + assertEquals(subsearches, multisearch.getChild()); + } + + @Test + public void testMultisearchWithChild() { + UnresolvedPlan subsearch1 = relation("table1"); + UnresolvedPlan subsearch2 = relation("table2"); + List subsearches = ImmutableList.of(subsearch1, subsearch2); + + Multisearch multisearch = new Multisearch(subsearches); + UnresolvedPlan mainQuery = relation("main_table"); + + // Attach child + multisearch.attach(mainQuery); + + // Verify child is attached + List children = multisearch.getChild(); + assertEquals(3, children.size()); // main query + 2 subsearches + assertEquals(mainQuery, children.get(0)); + assertEquals(subsearch1, children.get(1)); + assertEquals(subsearch2, children.get(2)); + } + + @Test + public void testMultisearchVisitorAccept() { + UnresolvedPlan subsearch = relation("table"); + Multisearch multisearch = new Multisearch(ImmutableList.of(subsearch)); + + // Test visitor pattern + TestVisitor visitor = new TestVisitor(); + String result = multisearch.accept(visitor, "test_context"); + + assertEquals("visitMultisearch_called_with_test_context", result); + } + + @Test + public void testMultisearchEqualsAndHashCode() { + UnresolvedPlan subsearch1 = relation("table1"); + UnresolvedPlan subsearch2 = relation("table2"); + List subsearches = ImmutableList.of(subsearch1, subsearch2); + + Multisearch multisearch1 = new Multisearch(subsearches); + Multisearch multisearch2 = new Multisearch(subsearches); + + assertEquals(multisearch1, multisearch2); + assertEquals(multisearch1.hashCode(), multisearch2.hashCode()); + } + + @Test + public void testMultisearchToString() { + UnresolvedPlan subsearch = relation("table"); + Multisearch multisearch = new Multisearch(ImmutableList.of(subsearch)); + + String toString = multisearch.toString(); + assertNotNull(toString); + assertTrue(toString.contains("Multisearch")); + } + + // Test visitor implementation + private static class TestVisitor extends AbstractNodeVisitor { + @Override + public String visitMultisearch(Multisearch node, String context) { + return "visitMultisearch_called_with_" + context; + } + } +} diff --git a/core/src/test/java/org/opensearch/sql/ast/tree/TimechartTest.java b/core/src/test/java/org/opensearch/sql/ast/tree/TimechartTest.java new file mode 100644 index 00000000000..d587ff71787 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/ast/tree/TimechartTest.java @@ -0,0 +1,241 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.opensearch.sql.ast.dsl.AstDSL.aggregate; +import static org.opensearch.sql.ast.dsl.AstDSL.alias; +import static org.opensearch.sql.ast.dsl.AstDSL.doubleLiteral; +import static org.opensearch.sql.ast.dsl.AstDSL.field; +import static org.opensearch.sql.ast.dsl.AstDSL.function; +import static org.opensearch.sql.ast.dsl.AstDSL.intLiteral; +import static org.opensearch.sql.ast.dsl.AstDSL.relation; + +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.opensearch.sql.ast.dsl.AstDSL; +import org.opensearch.sql.ast.expression.AggregateFunction; +import org.opensearch.sql.ast.expression.Let; +import org.opensearch.sql.ast.expression.Span; +import org.opensearch.sql.ast.expression.SpanUnit; +import org.opensearch.sql.ast.expression.UnresolvedExpression; + +class TimechartTest { + + /** + * @return test sources for per_* function test. + */ + private static Stream perFuncTestSources() { + return Stream.of( + Arguments.of(30, "s", "SECOND"), + Arguments.of(5, "m", "MINUTE"), + Arguments.of(2, "h", "HOUR"), + Arguments.of(1, "d", "DAY"), + Arguments.of(1, "w", "WEEK"), + Arguments.of(1, "M", "MONTH"), + Arguments.of(1, "q", "QUARTER"), + Arguments.of(1, "y", "YEAR")); + } + + @ParameterizedTest + @MethodSource("perFuncTestSources") + void should_transform_per_second_for_different_spans( + int spanValue, String spanUnit, String expectedIntervalUnit) { + withTimechart(span(spanValue, spanUnit), perSecond("bytes")) + .whenTransformingPerFunction() + .thenExpect( + eval( + let( + "per_second(bytes)", + divide( + multiply("per_second(bytes)", 1000.0), + timestampdiff( + "MILLISECOND", + "@timestamp", + timestampadd(expectedIntervalUnit, spanValue, "@timestamp")))), + timechart(span(spanValue, spanUnit), alias("per_second(bytes)", sum("bytes"))))); + } + + @ParameterizedTest + @MethodSource("perFuncTestSources") + void should_transform_per_minute_for_different_spans( + int spanValue, String spanUnit, String expectedIntervalUnit) { + withTimechart(span(spanValue, spanUnit), perMinute("bytes")) + .whenTransformingPerFunction() + .thenExpect( + eval( + let( + "per_minute(bytes)", + divide( + multiply("per_minute(bytes)", 60000.0), + timestampdiff( + "MILLISECOND", + "@timestamp", + timestampadd(expectedIntervalUnit, spanValue, "@timestamp")))), + timechart(span(spanValue, spanUnit), alias("per_minute(bytes)", sum("bytes"))))); + } + + @ParameterizedTest + @MethodSource("perFuncTestSources") + void should_transform_per_hour_for_different_spans( + int spanValue, String spanUnit, String expectedIntervalUnit) { + withTimechart(span(spanValue, spanUnit), perHour("bytes")) + .whenTransformingPerFunction() + .thenExpect( + eval( + let( + "per_hour(bytes)", + divide( + multiply("per_hour(bytes)", 3600000.0), + timestampdiff( + "MILLISECOND", + "@timestamp", + timestampadd(expectedIntervalUnit, spanValue, "@timestamp")))), + timechart(span(spanValue, spanUnit), alias("per_hour(bytes)", sum("bytes"))))); + } + + @ParameterizedTest + @MethodSource("perFuncTestSources") + void should_transform_per_day_for_different_spans( + int spanValue, String spanUnit, String expectedIntervalUnit) { + withTimechart(span(spanValue, spanUnit), perDay("bytes")) + .whenTransformingPerFunction() + .thenExpect( + eval( + let( + "per_day(bytes)", + divide( + multiply("per_day(bytes)", 8.64E7), + timestampdiff( + "MILLISECOND", + "@timestamp", + timestampadd(expectedIntervalUnit, spanValue, "@timestamp")))), + timechart(span(spanValue, spanUnit), alias("per_day(bytes)", sum("bytes"))))); + } + + @Test + void should_not_transform_non_per_functions() { + withTimechart(span(1, "m"), sum("bytes")) + .whenTransformingPerFunction() + .thenExpect(timechart(span(1, "m"), sum("bytes"))); + } + + @Test + void should_preserve_all_fields_during_per_function_transformation() { + Timechart original = + new Timechart(relation("logs"), perSecond("bytes")) + .span(span(5, "m")) + .by(field("status")) + .limit(20) + .useOther(false); + + Timechart expected = + new Timechart(relation("logs"), alias("per_second(bytes)", sum("bytes"))) + .span(span(5, "m")) + .by(field("status")) + .limit(20) + .useOther(false); + + withTimechart(original) + .whenTransformingPerFunction() + .thenExpect( + eval( + let( + "per_second(bytes)", + divide( + multiply("per_second(bytes)", 1000.0), + timestampdiff( + "MILLISECOND", "@timestamp", timestampadd("MINUTE", 5, "@timestamp")))), + expected)); + } + + // Fluent API for readable test assertions + + private static TransformationAssertion withTimechart(Span spanExpr, AggregateFunction aggFunc) { + return new TransformationAssertion(timechart(spanExpr, aggFunc)); + } + + private static TransformationAssertion withTimechart(Timechart timechart) { + return new TransformationAssertion(timechart); + } + + private static Timechart timechart(Span spanExpr, UnresolvedExpression aggExpr) { + // Set child here because expected object won't call attach below + return new Timechart(relation("t"), aggExpr).span(spanExpr).limit(10).useOther(true); + } + + private static Span span(int value, String unit) { + return AstDSL.span(field("@timestamp"), intLiteral(value), SpanUnit.of(unit)); + } + + private static AggregateFunction perSecond(String fieldName) { + return (AggregateFunction) aggregate("per_second", field(fieldName)); + } + + private static AggregateFunction perMinute(String fieldName) { + return (AggregateFunction) aggregate("per_minute", field(fieldName)); + } + + private static AggregateFunction perHour(String fieldName) { + return (AggregateFunction) aggregate("per_hour", field(fieldName)); + } + + private static AggregateFunction perDay(String fieldName) { + return (AggregateFunction) aggregate("per_day", field(fieldName)); + } + + private static AggregateFunction sum(String fieldName) { + return (AggregateFunction) aggregate("sum", field(fieldName)); + } + + private static Let let(String fieldName, UnresolvedExpression expression) { + return AstDSL.let(field(fieldName), expression); + } + + private static UnresolvedExpression multiply(String fieldName, double right) { + return function("*", field(fieldName), doubleLiteral(right)); + } + + private static UnresolvedExpression divide( + UnresolvedExpression left, UnresolvedExpression right) { + return function("/", left, right); + } + + private static UnresolvedExpression timestampadd(String unit, int value, String timestampField) { + return function( + "timestampadd", AstDSL.stringLiteral(unit), intLiteral(value), field(timestampField)); + } + + private static UnresolvedExpression timestampdiff( + String unit, String startField, UnresolvedExpression end) { + return function("timestampdiff", AstDSL.stringLiteral(unit), field(startField), end); + } + + private static UnresolvedPlan eval(Let letExpr, Timechart timechartExpr) { + return AstDSL.eval(timechartExpr, letExpr); + } + + private static class TransformationAssertion { + private final Timechart timechart; + private UnresolvedPlan result; + + TransformationAssertion(Timechart timechart) { + this.timechart = timechart; + } + + public TransformationAssertion whenTransformingPerFunction() { + this.result = timechart.attach(timechart.getChild().get(0)); + return this; + } + + public void thenExpect(UnresolvedPlan expected) { + assertEquals(expected, result); + } + } +} diff --git a/core/src/test/java/org/opensearch/sql/calcite/CalciteRexNodeVisitorTest.java b/core/src/test/java/org/opensearch/sql/calcite/CalciteRexNodeVisitorTest.java index ca9deb77061..1ec8d005716 100644 --- a/core/src/test/java/org/opensearch/sql/calcite/CalciteRexNodeVisitorTest.java +++ b/core/src/test/java/org/opensearch/sql/calcite/CalciteRexNodeVisitorTest.java @@ -68,7 +68,7 @@ public void setUpContext() { mockedStatic.when(() -> CalciteToolsHelper.create(any(), any(), any())).thenReturn(relBuilder); - context = CalcitePlanContext.create(frameworkConfig, 100, QueryType.PPL); + context = CalcitePlanContext.create(frameworkConfig, SysLimit.DEFAULT, QueryType.PPL); } @AfterEach diff --git a/core/src/test/java/org/opensearch/sql/calcite/PPLAggGroupMergeRuleTest.java b/core/src/test/java/org/opensearch/sql/calcite/PPLAggGroupMergeRuleTest.java new file mode 100644 index 00000000000..4960d9c73b6 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/calcite/PPLAggGroupMergeRuleTest.java @@ -0,0 +1,176 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.TYPE_FACTORY; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.plan.volcano.VolcanoRuleCall; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.tools.RelBuilder.AggCall; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.calcite.plan.OpenSearchRules; +import org.opensearch.sql.calcite.plan.PPLAggGroupMergeRule; +import org.opensearch.sql.calcite.utils.CalciteToolsHelper.OpenSearchRelBuilder; + +@ExtendWith(MockitoExtension.class) +public class PPLAggGroupMergeRuleTest { + @Mock VolcanoRuleCall mockedCall; + @Mock RelNode input; + @Mock RelOptCluster cluster; + @Mock RelOptPlanner planner; + @Mock RelMetadataQuery mq; + RelDataType type = TYPE_FACTORY.createSqlType(SqlTypeName.BIGINT); + RelDataType rowType = TYPE_FACTORY.createStructType(List.of(type, type), List.of("a", "b")); + RexBuilder rexBuilder = new RexBuilder(TYPE_FACTORY); + RelBuilder relBuilder; + + @BeforeEach + public void setUp() throws IllegalAccessException, NoSuchFieldException { + when(cluster.getTypeFactory()).thenReturn(TYPE_FACTORY); + when(cluster.getRexBuilder()).thenReturn(rexBuilder); + lenient().when(mq.isVisibleInExplain(any(), any())).thenReturn(true); + when(cluster.getMetadataQuery()).thenReturn(mq); + when(cluster.traitSet()).thenReturn(RelTraitSet.createEmpty()); + when(cluster.traitSetOf(Convention.NONE)) + .thenReturn(RelTraitSet.createEmpty().replace(Convention.NONE)); + when(cluster.getPlanner()).thenReturn(planner); + when(planner.getExecutor()).thenReturn(null); + + when(input.getCluster()).thenReturn(cluster); + when(input.getRowType()).thenReturn(rowType); + relBuilder = new OpenSearchRelBuilder(null, cluster, null); + lenient().when(mockedCall.builder()).thenReturn(relBuilder); + } + + @Test + public void testRuleMatch() { + relBuilder.push(input); + RexNode baseGroupField = new RexInputRef(0, TYPE_FACTORY.createSqlType(SqlTypeName.BIGINT)); + RexNode dependentGroupField = + rexBuilder.makeCall( + SqlStdOperatorTable.PLUS, List.of(baseGroupField, rexBuilder.makeLiteral(10, type))); + AggCall aggCall = relBuilder.aggregateCall(SqlStdOperatorTable.COUNT); + LogicalAggregate aggregate = + (LogicalAggregate) + relBuilder + .aggregate( + relBuilder.groupKey(baseGroupField, dependentGroupField), + ImmutableList.of(aggCall)) + .build(); + assert (aggregate.getInput() instanceof LogicalProject); + LogicalProject project = (LogicalProject) aggregate.getInput(); + + // Check the predicate in Config + assertTrue(PPLAggGroupMergeRule.Config.containsMultipleGroupSets(aggregate)); + assertTrue(PPLAggGroupMergeRule.Config.containsDependentFields(project)); + + assertEquals( + "LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()])\n" + + " LogicalProject(a=[$0], $f2=[+($0, 10)])\n", + aggregate.explain().replaceAll("\\r\\n", "\n")); + + doAnswer( + invocation -> { + // Check the final plan + RelNode rel = invocation.getArgument(0); + assertTrue( + RelOptUtil.areRowTypesEqual(rel.getRowType(), aggregate.getRowType(), false)); + assertEquals( + "LogicalProject(a=[$0], $f1=[+($0, 10)], $f10=[$1])\n" + + " LogicalAggregate(group=[{0}], agg#0=[COUNT()])\n" + + " LogicalProject(a=[$0])\n", + rel.explain().replaceAll("\\r\\n", "\n")); + return null; + }) + .when(mockedCall) + .transformTo(any()); + OpenSearchRules.AGG_GROUP_MERGE_RULE.apply(mockedCall, aggregate, project); + } + + // TODO: May support this case in the future + @Test + public void testRuleMatch_NoMergeForMultipleBaseFields() { + relBuilder.push(input); + RexNode baseGroupField = new RexInputRef(0, TYPE_FACTORY.createSqlType(SqlTypeName.BIGINT)); + RexNode baseGroupField2 = new RexInputRef(1, TYPE_FACTORY.createSqlType(SqlTypeName.BIGINT)); + RexNode dependentGroupField = + rexBuilder.makeCall( + SqlStdOperatorTable.PLUS, List.of(baseGroupField, rexBuilder.makeLiteral(10, type))); + AggCall aggCall = relBuilder.aggregateCall(SqlStdOperatorTable.COUNT); + LogicalAggregate aggregate = + (LogicalAggregate) + relBuilder + .aggregate( + relBuilder.groupKey(baseGroupField, baseGroupField2, dependentGroupField), + ImmutableList.of(aggCall)) + .build(); + assert (aggregate.getInput() instanceof LogicalProject); + LogicalProject project = (LogicalProject) aggregate.getInput(); + + // Check the predicate in Config + assertTrue(PPLAggGroupMergeRule.Config.containsMultipleGroupSets(aggregate)); + assertTrue(PPLAggGroupMergeRule.Config.containsDependentFields(project)); + + assertEquals( + "LogicalAggregate(group=[{0, 1, 2}], agg#0=[COUNT()])\n" + + " LogicalProject(a=[$0], b=[$1], $f2=[+($0, 10)])\n", + aggregate.explain().replaceAll("\\r\\n", "\n")); + + OpenSearchRules.AGG_GROUP_MERGE_RULE.apply(mockedCall, aggregate, project); + // Assert don't invoke transformTo + verify(mockedCall, never()).transformTo(any()); + } + + // TODO: May support this case in the future + @Test + public void testRuleMatch_NoMatchForNoBaseFields() { + relBuilder.push(input); + RexNode baseGroupField = new RexInputRef(0, TYPE_FACTORY.createSqlType(SqlTypeName.BIGINT)); + RexNode dependentGroupField = + rexBuilder.makeCall( + SqlStdOperatorTable.PLUS, List.of(baseGroupField, rexBuilder.makeLiteral(10, type))); + AggCall aggCall = relBuilder.aggregateCall(SqlStdOperatorTable.COUNT); + LogicalAggregate aggregate = + (LogicalAggregate) + relBuilder + .aggregate(relBuilder.groupKey(dependentGroupField), ImmutableList.of(aggCall)) + .build(); + assert (aggregate.getInput() instanceof LogicalProject); + + // Check the predicate in Config + assertFalse(PPLAggGroupMergeRule.Config.containsMultipleGroupSets(aggregate)); + } +} diff --git a/core/src/test/java/org/opensearch/sql/calcite/PPLAggregateConvertRuleTest.java b/core/src/test/java/org/opensearch/sql/calcite/PPLAggregateConvertRuleTest.java index 652ee108aca..1086fc5f0c1 100644 --- a/core/src/test/java/org/opensearch/sql/calcite/PPLAggregateConvertRuleTest.java +++ b/core/src/test/java/org/opensearch/sql/calcite/PPLAggregateConvertRuleTest.java @@ -5,7 +5,7 @@ package org.opensearch.sql.calcite; -import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; @@ -23,9 +23,9 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.fun.SqlStdOperatorTable; @@ -37,17 +37,17 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.calcite.plan.OpenSearchRules; import org.opensearch.sql.calcite.plan.PPLAggregateConvertRule; import org.opensearch.sql.calcite.utils.CalciteToolsHelper.OpenSearchRelBuilder; @ExtendWith(MockitoExtension.class) public class PPLAggregateConvertRuleTest { - public static final PPLAggregateConvertRule AGGREGATE_CONVERT_RULE = - PPLAggregateConvertRule.Config.SUM_CONVERTER.toRule(); @Mock VolcanoRuleCall mockedCall; @Mock RelNode input; @Mock RelOptCluster cluster; @Mock RelOptPlanner planner; + @Mock RelMetadataQuery mq; RelDataType type = TYPE_FACTORY.createSqlType(SqlTypeName.BIGINT); RelDataType rowType = TYPE_FACTORY.createStructType(List.of(type, type), List.of("a", "b")); RexBuilder rexBuilder = new RexBuilder(TYPE_FACTORY); @@ -57,6 +57,8 @@ public class PPLAggregateConvertRuleTest { public void setUp() throws IllegalAccessException, NoSuchFieldException { when(cluster.getTypeFactory()).thenReturn(TYPE_FACTORY); when(cluster.getRexBuilder()).thenReturn(rexBuilder); + when(mq.isVisibleInExplain(any(), any())).thenReturn(true); + when(cluster.getMetadataQuery()).thenReturn(mq); when(cluster.traitSet()).thenReturn(RelTraitSet.createEmpty()); when(cluster.traitSetOf(Convention.NONE)) .thenReturn(RelTraitSet.createEmpty().replace(Convention.NONE)); @@ -89,36 +91,26 @@ public void testRuleMatch() { assertTrue(PPLAggregateConvertRule.Config.containsSumAggCall(aggregate)); assertTrue(PPLAggregateConvertRule.Config.containsCallWithNumber(project)); + assertEquals( + "LogicalAggregate(group=[{0}], agg#0=[SUM($1)])\n" + + " LogicalProject(b=[$1], $f2=[+($0, 10)])\n", + aggregate.explain().replaceAll("\\r\\n", "\n")); doAnswer( invocation -> { // Check the final plan RelNode rel = invocation.getArgument(0); assertTrue( - RelOptUtil.areRowTypesEqual(rel.getRowType(), aggregate.getRowType(), true)); - - assertInstanceOf(LogicalProject.class, rel); - LogicalProject parentProject = (LogicalProject) rel; - assertTrue( - parentProject.getProjects().getLast() instanceof RexCall call - && call.getOperator() == SqlStdOperatorTable.PLUS); - - assertInstanceOf(LogicalAggregate.class, parentProject.getInput()); - LogicalAggregate newAggregate = (LogicalAggregate) parentProject.getInput(); - assertTrue( - newAggregate.getAggCallList().getFirst().getAggregation() - == SqlStdOperatorTable.SUM - && newAggregate.getAggCallList().getLast().getAggregation() - == SqlStdOperatorTable.COUNT); - - assertInstanceOf(LogicalProject.class, newAggregate.getInput()); - LogicalProject childProject = (LogicalProject) newAggregate.getInput(); - assertTrue( - childProject.getProjects().stream().allMatch(rex -> rex instanceof RexInputRef)); - + RelOptUtil.areRowTypesEqual(rel.getRowType(), aggregate.getRowType(), false)); + assertEquals( + "LogicalProject(b=[$0], $f1=[+($1, *($2, 10))])\n" + + " LogicalAggregate(group=[{0}], null_SUM=[SUM($1)]," + + " null_COUNT=[COUNT($1)])\n" + + " LogicalProject(b=[$1], a=[$0])\n", + rel.explain().replaceAll("\\r\\n", "\n")); return null; }) .when(mockedCall) .transformTo(any()); - AGGREGATE_CONVERT_RULE.apply(mockedCall, aggregate, project); + OpenSearchRules.AGGREGATE_CONVERT_RULE.apply(mockedCall, aggregate, project); } } diff --git a/core/src/test/java/org/opensearch/sql/calcite/utils/WildcardUtilsTest.java b/core/src/test/java/org/opensearch/sql/calcite/utils/WildcardUtilsTest.java index 53cc1d5163c..2e41de018a5 100644 --- a/core/src/test/java/org/opensearch/sql/calcite/utils/WildcardUtilsTest.java +++ b/core/src/test/java/org/opensearch/sql/calcite/utils/WildcardUtilsTest.java @@ -5,6 +5,11 @@ package org.opensearch.sql.calcite.utils; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + import com.google.common.collect.ImmutableList; import java.util.List; import org.junit.jupiter.api.BeforeEach; @@ -74,6 +79,32 @@ void testMatchesWildcardPattern() { testPattern("*a*e", "city", false); } + @Test + void testMatchesWildcardPatternEdgeCases() { + // Test null handling + assertFalse(WildcardUtils.matchesWildcardPattern(null, "field")); + assertFalse(WildcardUtils.matchesWildcardPattern("pattern", null)); + assertFalse(WildcardUtils.matchesWildcardPattern(null, null)); + + // Test empty strings + assertTrue(WildcardUtils.matchesWildcardPattern("", "")); + assertFalse(WildcardUtils.matchesWildcardPattern("", "field")); + assertFalse(WildcardUtils.matchesWildcardPattern("field", "")); + + // Test single wildcard + assertTrue(WildcardUtils.matchesWildcardPattern("*", "anything")); + assertTrue(WildcardUtils.matchesWildcardPattern("*", "")); + + // Test multiple consecutive wildcards + assertTrue(WildcardUtils.matchesWildcardPattern("**", "field")); + assertTrue(WildcardUtils.matchesWildcardPattern("a**b", "ab")); + assertTrue(WildcardUtils.matchesWildcardPattern("a**b", "axxxb")); + + // Test wildcards at start and end + assertTrue(WildcardUtils.matchesWildcardPattern("*field*", "myfield123")); + assertTrue(WildcardUtils.matchesWildcardPattern("*field*", "field")); + } + @Test void testExpandWildcardPattern() { // Test exact match @@ -97,6 +128,20 @@ void testExpandWildcardPattern() { testExpansion("XYZ*", ImmutableList.of()); } + @Test + void testExpandWildcardPatternEdgeCases() { + // Test null handling + assertEquals(List.of(), WildcardUtils.expandWildcardPattern(null, availableFields)); + assertEquals(List.of(), WildcardUtils.expandWildcardPattern("pattern", null)); + assertEquals(List.of(), WildcardUtils.expandWildcardPattern(null, null)); + + // Test empty list + assertEquals(List.of(), WildcardUtils.expandWildcardPattern("*", List.of())); + + // Test single wildcard matches all + assertEquals(availableFields, WildcardUtils.expandWildcardPattern("*", availableFields)); + } + @Test void testContainsWildcard() { // Test with wildcard @@ -108,4 +153,142 @@ void testContainsWildcard() { testContainsWildcard("field", false); testContainsWildcard("", false); } + + @Test + void testContainsWildcardEdgeCases() { + // Test null + assertFalse(WildcardUtils.containsWildcard(null)); + + // Test multiple wildcards + assertTrue(WildcardUtils.containsWildcard("**")); + assertTrue(WildcardUtils.containsWildcard("a*b*c")); + } + + @Test + void testConvertWildcardPatternToRegex() { + // Basic patterns + assertEquals("^\\Qada\\E$", WildcardUtils.convertWildcardPatternToRegex("ada")); + assertEquals("^\\Q\\E(.*?)\\Qada\\E$", WildcardUtils.convertWildcardPatternToRegex("*ada")); + assertEquals("^\\Qada\\E(.*?)\\Q\\E$", WildcardUtils.convertWildcardPatternToRegex("ada*")); + assertEquals( + "^\\Q\\E(.*?)\\Qada\\E(.*?)\\Q\\E$", WildcardUtils.convertWildcardPatternToRegex("*ada*")); + + // Multiple wildcards + assertEquals( + "^\\Qa\\E(.*?)\\Qb\\E(.*?)\\Qc\\E$", WildcardUtils.convertWildcardPatternToRegex("a*b*c")); + + // Pattern with special regex characters + assertEquals( + "^\\Qa.b\\E(.*?)\\Qc+d\\E$", WildcardUtils.convertWildcardPatternToRegex("a.b*c+d")); + + // Single wildcard + assertEquals("^\\Q\\E(.*?)\\Q\\E$", WildcardUtils.convertWildcardPatternToRegex("*")); + + // Empty pattern + assertEquals("^\\Q\\E$", WildcardUtils.convertWildcardPatternToRegex("")); + + // Invalid pattern with trailing backslash should throw + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, + () -> WildcardUtils.convertWildcardPatternToRegex("pattern\\")); + assertTrue(ex.getMessage().contains("Invalid escape sequence")); + } + + @Test + void testConvertWildcardReplacementToRegex() { + // No wildcards - literal replacement + assertEquals("ada", WildcardUtils.convertWildcardReplacementToRegex("ada")); + assertEquals("test_value", WildcardUtils.convertWildcardReplacementToRegex("test_value")); + + // Single wildcard + assertEquals("$1", WildcardUtils.convertWildcardReplacementToRegex("*")); + + // Wildcards with text + assertEquals("$1_$2", WildcardUtils.convertWildcardReplacementToRegex("*_*")); + assertEquals("prefix_$1", WildcardUtils.convertWildcardReplacementToRegex("prefix_*")); + assertEquals("$1_suffix", WildcardUtils.convertWildcardReplacementToRegex("*_suffix")); + + // Multiple wildcards + assertEquals("$1_$2_$3", WildcardUtils.convertWildcardReplacementToRegex("*_*_*")); + + // Empty string + assertEquals("", WildcardUtils.convertWildcardReplacementToRegex("")); + } + + @Test + void testConvertWildcardReplacementToRegexWithEscapes() { + // Escaped wildcard should be treated as literal + assertEquals("*", WildcardUtils.convertWildcardReplacementToRegex("\\*")); // \* -> * + assertEquals("$1_*", WildcardUtils.convertWildcardReplacementToRegex("*_\\*")); + assertEquals("*_$1", WildcardUtils.convertWildcardReplacementToRegex("\\*_*")); + + // Escaped backslash when there's no wildcard - returned unchanged + assertEquals("\\\\", WildcardUtils.convertWildcardReplacementToRegex("\\\\")); + + // Mixed escaped and unescaped wildcards + assertEquals("$1_*_$2", WildcardUtils.convertWildcardReplacementToRegex("*_\\*_*")); + assertEquals("$1\\$2", WildcardUtils.convertWildcardReplacementToRegex("*\\\\*")); // \\ -> \ + } + + @Test + void testValidateWildcardSymmetry() { + // Valid: same number of wildcards + WildcardUtils.validateWildcardSymmetry("*", "*"); + WildcardUtils.validateWildcardSymmetry("*ada*", "*_*"); + WildcardUtils.validateWildcardSymmetry("a*b*c", "x*y*z"); + + // Valid: replacement has no wildcards (literal replacement) + WildcardUtils.validateWildcardSymmetry("*", "literal"); + WildcardUtils.validateWildcardSymmetry("*ada*", "replacement"); + WildcardUtils.validateWildcardSymmetry("a*b*c", "xyz"); + + // Valid: pattern has no wildcards + WildcardUtils.validateWildcardSymmetry("ada", "replacement"); + } + + @Test + void testValidateWildcardSymmetryFailure() { + // Invalid: mismatched wildcard counts + IllegalArgumentException ex1 = + assertThrows( + IllegalArgumentException.class, + () -> WildcardUtils.validateWildcardSymmetry("*", "**")); + assertTrue(ex1.getMessage().contains("Wildcard count mismatch")); + assertTrue(ex1.getMessage().contains("pattern has 1 wildcard(s)")); + assertTrue(ex1.getMessage().contains("replacement has 2")); + + IllegalArgumentException ex2 = + assertThrows( + IllegalArgumentException.class, + () -> WildcardUtils.validateWildcardSymmetry("*a*b*", "*_*")); + assertTrue(ex2.getMessage().contains("pattern has 3 wildcard(s)")); + assertTrue(ex2.getMessage().contains("replacement has 2")); + + IllegalArgumentException ex3 = + assertThrows( + IllegalArgumentException.class, + () -> WildcardUtils.validateWildcardSymmetry("ada", "*")); + assertTrue(ex3.getMessage().contains("pattern has 0 wildcard(s)")); + assertTrue(ex3.getMessage().contains("replacement has 1")); + } + + @Test + void testValidateWildcardSymmetryWithEscapes() { + // Escaped wildcards should not count + WildcardUtils.validateWildcardSymmetry("\\*", "literal"); // 0 wildcards in pattern + WildcardUtils.validateWildcardSymmetry("*\\*", "*"); // 1 wildcard in both + + // Pattern with 2 wildcards, replacement with 1 wildcard (middle one in \\**\\*) + WildcardUtils.validateWildcardSymmetry("*", "\\**\\*"); // 1 wildcard in both + + // Should fail when unescaped counts don't match + assertThrows( + IllegalArgumentException.class, + () -> WildcardUtils.validateWildcardSymmetry("*a*", "*\\*")); // 2 vs 1 + + assertThrows( + IllegalArgumentException.class, + () -> WildcardUtils.validateWildcardSymmetry("*a*", "\\**\\*")); // 2 vs 1 + } } diff --git a/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java b/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java index 727fefcff7c..77a5081bb5b 100644 --- a/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java @@ -103,6 +103,7 @@ public Helper() { lenient().when(analyzer.analyze(any(), any())).thenReturn(logicalPlan); lenient().when(planner.plan(any())).thenReturn(plan); lenient().when(settings.getSettingValue(Key.QUERY_SIZE_LIMIT)).thenReturn(200); + lenient().when(settings.getSettingValue(Key.QUERY_BUCKET_SIZE)).thenReturn(1000); lenient().when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(false); queryService = new QueryService(analyzer, executionEngine, planner, null, settings); diff --git a/core/src/test/java/org/opensearch/sql/expression/function/CoercionUtilsTest.java b/core/src/test/java/org/opensearch/sql/expression/function/CoercionUtilsTest.java new file mode 100644 index 00000000000..30d827f1ecc --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/function/CoercionUtilsTest.java @@ -0,0 +1,107 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function; + +import static org.junit.jupiter.api.Assertions.*; +import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; +import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.util.List; +import java.util.stream.Stream; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; + +class CoercionUtilsTest { + + private static final RexBuilder REX_BUILDER = new RexBuilder(OpenSearchTypeFactory.TYPE_FACTORY); + + private static RexNode nullLiteral(ExprCoreType type) { + return REX_BUILDER.makeNullLiteral(OpenSearchTypeFactory.convertExprTypeToRelDataType(type)); + } + + private static Stream commonWidestTypeArguments() { + return Stream.of( + Arguments.of(STRING, INTEGER, DOUBLE), + Arguments.of(INTEGER, STRING, DOUBLE), + Arguments.of(STRING, DOUBLE, DOUBLE), + Arguments.of(INTEGER, BOOLEAN, null)); + } + + @ParameterizedTest + @MethodSource("commonWidestTypeArguments") + public void findCommonWidestType( + ExprCoreType left, ExprCoreType right, ExprCoreType expectedCommonType) { + assertEquals( + expectedCommonType, CoercionUtils.resolveCommonType(left, right).orElseGet(() -> null)); + } + + @Test + void castArgumentsReturnsExactMatchWhenAvailable() { + PPLTypeChecker typeChecker = new StubTypeChecker(List.of(List.of(INTEGER), List.of(DOUBLE))); + List arguments = List.of(nullLiteral(INTEGER)); + + List result = CoercionUtils.castArguments(REX_BUILDER, typeChecker, arguments); + + assertNotNull(result); + assertEquals(1, result.size()); + assertEquals( + INTEGER, OpenSearchTypeFactory.convertRelDataTypeToExprType(result.getFirst().getType())); + } + + @Test + void castArgumentsFallsBackToWidestCandidate() { + PPLTypeChecker typeChecker = + new StubTypeChecker(List.of(List.of(ExprCoreType.LONG), List.of(DOUBLE))); + List arguments = List.of(nullLiteral(STRING)); + + List result = CoercionUtils.castArguments(REX_BUILDER, typeChecker, arguments); + + assertNotNull(result); + assertEquals( + DOUBLE, OpenSearchTypeFactory.convertRelDataTypeToExprType(result.getFirst().getType())); + } + + @Test + void castArgumentsReturnsNullWhenNoCompatibleSignatureExists() { + PPLTypeChecker typeChecker = new StubTypeChecker(List.of(List.of(ExprCoreType.GEO_POINT))); + List arguments = List.of(nullLiteral(INTEGER)); + + assertNull(CoercionUtils.castArguments(REX_BUILDER, typeChecker, arguments)); + } + + private static class StubTypeChecker implements PPLTypeChecker { + private final List> signatures; + + private StubTypeChecker(List> signatures) { + this.signatures = signatures; + } + + @Override + public boolean checkOperandTypes(List types) { + return false; + } + + @Override + public String getAllowedSignatures() { + return ""; + } + + @Override + public List> getParameterTypes() { + return signatures; + } + } +} diff --git a/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImplTest.java b/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImplTest.java new file mode 100644 index 00000000000..31fda119961 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImplTest.java @@ -0,0 +1,82 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.CollectionUDF; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.Test; + +/** Unit tests for MVAppendFunctionImpl. */ +public class MVAppendFunctionImplTest { + + @Test + public void testMvappendWithNoArguments() { + Object result = MVAppendFunctionImpl.mvappend(); + assertNull(result); + } + + @Test + public void testMvappendWithSingleElement() { + Object result = MVAppendFunctionImpl.mvappend(42); + assertEquals(Arrays.asList(42), result); + } + + @Test + public void testMvappendWithMultipleElements() { + Object result = MVAppendFunctionImpl.mvappend(1, 2, 3); + assertEquals(Arrays.asList(1, 2, 3), result); + } + + @Test + public void testMvappendWithNullValues() { + Object result = MVAppendFunctionImpl.mvappend(null, 1, null); + assertEquals(Arrays.asList(1), result); + } + + @Test + public void testMvappendWithAllNulls() { + Object result = MVAppendFunctionImpl.mvappend(null, null); + assertNull(result); + } + + @Test + public void testMvappendWithArrayFlattening() { + List array1 = Arrays.asList(1, 2); + List array2 = Arrays.asList(3, 4); + Object result = MVAppendFunctionImpl.mvappend(array1, array2); + assertEquals(Arrays.asList(1, 2, 3, 4), result); + } + + @Test + public void testMvappendWithMixedTypes() { + List array = Arrays.asList(1, 2); + Object result = MVAppendFunctionImpl.mvappend(array, 3, "hello"); + assertEquals(Arrays.asList(1, 2, 3, "hello"), result); + } + + @Test + public void testMvappendWithArrayAndNulls() { + List array = Arrays.asList(1, 2); + Object result = MVAppendFunctionImpl.mvappend(null, array, null, 3); + assertEquals(Arrays.asList(1, 2, 3), result); + } + + @Test + public void testMvappendWithSingleNull() { + Object result = MVAppendFunctionImpl.mvappend((Object) null); + assertNull(result); + } + + @Test + public void testMvappendWithEmptyArray() { + List emptyArray = Arrays.asList(); + Object result = MVAppendFunctionImpl.mvappend(emptyArray, 1); + assertEquals(Arrays.asList(1), result); + } +} diff --git a/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/MapAppendFunctionImplTest.java b/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/MapAppendFunctionImplTest.java new file mode 100644 index 00000000000..28df3c3cbd1 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/MapAppendFunctionImplTest.java @@ -0,0 +1,108 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.CollectionUDF; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class MapAppendFunctionImplTest { + @Test + void testMapAppendWithNonOverlappingKeys() { + Map map1 = getMap1(); + Map map2 = getMap2(); + + Map result = MapAppendFunctionImpl.mapAppendImpl(map1, map2); + + assertEquals(4, result.size()); + assertMapListValues(result, "a", "value1"); + assertMapListValues(result, "b", "value2"); + assertMapListValues(result, "c", "value3"); + assertMapListValues(result, "d", "value4"); + } + + @Test + void testMapAppendWithOverlappingKeys() { + Map map1 = getMap1(); + Map map2 = Map.of("b", "value3", "c", "value4"); + + Map result = MapAppendFunctionImpl.mapAppendImpl(map1, map2); + + assertEquals(3, result.size()); + assertMapListValues(result, "a", "value1"); + assertMapListValues(result, "b", "value2", "value3"); + assertMapListValues(result, "c", "value4"); + } + + @Test + void testMapAppendWithArrayValues() { + Map map1 = Map.of("a", List.of("item1", "item2"), "b", "single"); + Map map2 = Map.of("a", "item3", "c", List.of("item4", "item5")); + + Map result = MapAppendFunctionImpl.mapAppendImpl(map1, map2); + + assertEquals(3, result.size()); + assertMapListValues(result, "a", "item1", "item2", "item3"); + assertMapListValues(result, "b", "single"); + assertMapListValues(result, "c", "item4", "item5"); + } + + @Test + void testMapAppendWithNullValues() { + Map map1 = getMap1(); + map1.put("b", null); + Map map2 = getMap2(); + map2.put("b", "value2"); + map2.put("a", null); + + Map result = MapAppendFunctionImpl.mapAppendImpl(map1, map2); + + assertEquals(4, result.size()); + assertMapListValues(result, "a", "value1"); + assertMapListValues(result, "b", "value2"); + assertMapListValues(result, "c", "value3"); + assertMapListValues(result, "d", "value4"); + } + + @Test + void testMapAppendWithSingleParam() { + Map map1 = getMap1(); + + Map result = MapAppendFunctionImpl.mapAppendImpl(map1); + + assertEquals(2, result.size()); + assertMapListValues(result, "a", "value1"); + assertMapListValues(result, "b", "value2"); + } + + private Map getMap1() { + Map map1 = new HashMap<>(); + map1.put("a", "value1"); + map1.put("b", "value2"); + return map1; + } + + private Map getMap2() { + Map map2 = new HashMap<>(); + map2.put("c", "value3"); + map2.put("d", "value4"); + return map2; + } + + private void assertMapListValues(Map map, String key, Object... expectedValues) { + Object val = map.get(key); + assertTrue(val instanceof List); + List result = (List) val; + assertEquals(expectedValues.length, result.size()); + for (int i = 0; i < expectedValues.length; i++) { + assertEquals(expectedValues[i], result.get(i)); + } + } +} diff --git a/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/MapRemoveFunctionImplTest.java b/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/MapRemoveFunctionImplTest.java new file mode 100644 index 00000000000..f0fce0658d0 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/MapRemoveFunctionImplTest.java @@ -0,0 +1,205 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.CollectionUDF; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +public class MapRemoveFunctionImplTest { + + @Test + public void testMapRemoveWithNullMap() { + Object result = MapRemoveFunctionImpl.mapRemove(null, Arrays.asList("key1", "key2")); + assertNull(result); + } + + @Test + public void testMapRemoveWithNullKeys() { + Map map = getBaseMap(); + + Object result = MapRemoveFunctionImpl.mapRemove(map, null); + assertEquals(map, result); + } + + @Test + public void testMapRemoveWithInvalidMapArgument() { + String notAMap = "not a map"; + List keysToRemove = Arrays.asList("key1"); + + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> MapRemoveFunctionImpl.mapRemove(notAMap, keysToRemove)); + + assertEquals( + "First argument must be a map, got: class java.lang.String", exception.getMessage()); + } + + @Test + public void testMapRemoveWithInvalidKeysArgument() { + Map map = getBaseMap(); + String notAList = "not a list"; + + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, () -> MapRemoveFunctionImpl.mapRemove(map, notAList)); + + assertEquals( + "Second argument must be an array/list, got: class java.lang.String", + exception.getMessage()); + } + + @Test + public void testMapRemoveExistingKeys() { + Map map = getBaseMap(); + List keysToRemove = Arrays.asList("key1", "key3"); + + @SuppressWarnings("unchecked") + Map result = + (Map) MapRemoveFunctionImpl.mapRemove(map, keysToRemove); + + assertEquals(1, result.size()); + assertEquals("value2", result.get("key2")); + assertNull(result.get("key1")); + assertNull(result.get("key3")); + + // Verify original map is not modified + assertEqualToBaseMap(map); + } + + @Test + public void testMapRemoveNonExistingKeys() { + Map map = getBaseMap(); + List keysToRemove = Arrays.asList("key4", "key5"); + + @SuppressWarnings("unchecked") + Map result = + (Map) MapRemoveFunctionImpl.mapRemove(map, keysToRemove); + + assertEqualToBaseMap(result); + } + + @Test + public void testMapRemoveEmptyKeysList() { + Map map = getBaseMap(); + List keysToRemove = Arrays.asList(); + + @SuppressWarnings("unchecked") + Map result = + (Map) MapRemoveFunctionImpl.mapRemove(map, keysToRemove); + + assertEqualToBaseMap(result); + } + + @Test + public void testMapRemoveMixedExistingAndNonExistingKeys() { + Map map = getBaseMap(); + List keysToRemove = Arrays.asList("key1", "key4", "key2"); + + @SuppressWarnings("unchecked") + Map result = + (Map) MapRemoveFunctionImpl.mapRemove(map, keysToRemove); + + assertEquals(1, result.size()); + assertEquals("value3", result.get("key3")); + } + + @Test + public void testMapRemoveWithNullKeysInList() { + Map map = getBaseMap(); + List keysToRemove = Arrays.asList("key1", null, "key3"); + + @SuppressWarnings("unchecked") + Map result = + (Map) MapRemoveFunctionImpl.mapRemove(map, keysToRemove); + + assertEquals(1, result.size()); + assertEquals("value2", result.get("key2")); + } + + @Test + public void testMapRemoveWithDifferentValueTypes() { + Map map = new HashMap<>(); + map.put("string", "value"); + map.put("number", 42); + map.put("boolean", true); + map.put("list", Arrays.asList(1, 2, 3)); + List keysToRemove = Arrays.asList("number", "boolean"); + + @SuppressWarnings("unchecked") + Map result = + (Map) MapRemoveFunctionImpl.mapRemove(map, keysToRemove); + + assertEquals(2, result.size()); + assertEquals("value", result.get("string")); + assertEquals(Arrays.asList(1, 2, 3), result.get("list")); + assertNull(result.get("number")); + assertNull(result.get("boolean")); + } + + @Test + public void testMapRemoveAllKeys() { + Map map = getBaseMap(); + List keysToRemove = Arrays.asList("key1", "key2", "key3"); + + @SuppressWarnings("unchecked") + Map result = + (Map) MapRemoveFunctionImpl.mapRemove(map, keysToRemove); + + assertEquals(0, result.size()); + } + + @Test + public void testMapRemoveWithEmptyMap() { + Map map = new HashMap<>(); + List keysToRemove = Arrays.asList("key1", "key2"); + + @SuppressWarnings("unchecked") + Map result = + (Map) MapRemoveFunctionImpl.mapRemove(map, keysToRemove); + + assertEquals(0, result.size()); + } + + @Test + public void testMapRemoveWithNonStringKeys() { + Map map = new HashMap<>(); + map.put("key1", "value1"); + map.put("key2", "value2"); + map.put("123", "numeric_key_value"); + + List keysToRemove = Arrays.asList("key1", 123); // 123 will be converted to string "123" + + @SuppressWarnings("unchecked") + Map result = + (Map) MapRemoveFunctionImpl.mapRemove(map, keysToRemove); + + assertEquals(1, result.size()); + assertEquals("value2", result.get("key2")); + } + + private Map getBaseMap() { + Map map = new HashMap<>(); + map.put("key1", "value1"); + map.put("key2", "value2"); + map.put("key3", "value3"); + return map; + } + + private void assertEqualToBaseMap(Map map) { + assertEquals(3, map.size()); + assertEquals("value1", map.get("key1")); + assertEquals("value2", map.get("key2")); + assertEquals("value3", map.get("key3")); + } +} diff --git a/core/src/test/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractAllFunctionImplTest.java b/core/src/test/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractAllFunctionImplTest.java new file mode 100644 index 00000000000..5a010a17422 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractAllFunctionImplTest.java @@ -0,0 +1,360 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +public class JsonExtractAllFunctionImplTest { + + private final JsonExtractAllFunctionImpl function = new JsonExtractAllFunctionImpl(); + + @SuppressWarnings("unchecked") + private Map assertValidMapResult(Object result) { + assertNotNull(result); + assertTrue(result instanceof Map); + return (Map) result; + } + + @SuppressWarnings("unchecked") + private List assertListValue(Map map, String key) { + Object value = map.get(key); + assertNotNull(value); + assertTrue(value instanceof List); + return (List) value; + } + + private void assertListEquals(List actual, Object... expected) { + assertEquals(expected.length, actual.size()); + for (int i = 0; i < expected.length; i++) { + assertEquals(expected[i], actual.get(i)); + } + } + + private void assertMapListValue(Map map, String key, Object... expectedValues) { + List list = assertListValue(map, key); + assertListEquals(list, expectedValues); + } + + private void assertMapValue(Map map, String key, Object expectedValue) { + assertEquals(expectedValue, map.get(key)); + } + + private Map eval(String json) { + Object result = JsonExtractAllFunctionImpl.eval(json); + return assertValidMapResult(result); + } + + @Test + public void testReturnTypeInference() { + assertNotNull(function.getReturnTypeInference(), "Return type inference should not be null"); + } + + @Test + public void testOperandMetadata() { + assertNotNull(function.getOperandMetadata(), "Operand metadata should not be null"); + } + + @Test + public void testFunctionConstructor() { + JsonExtractAllFunctionImpl testFunction = new JsonExtractAllFunctionImpl(); + + assertNotNull(testFunction, "Function should be properly initialized"); + } + + @Test + public void testNoArguments() { + Object result = JsonExtractAllFunctionImpl.eval(); + + assertNull(result); + } + + @Test + public void testNullInput() { + Object result = JsonExtractAllFunctionImpl.eval((String) null); + + assertNull(result); + } + + @Test + public void testEmptyString() { + Object result = JsonExtractAllFunctionImpl.eval(""); + + assertNull(result); + } + + @Test + public void testWhitespaceString() { + Object result = JsonExtractAllFunctionImpl.eval(" "); + + assertNull(result); + } + + @Test + public void testEmptyJsonObject() { + Map map = eval("{}"); + + assertTrue(map.isEmpty()); + } + + @Test + public void testSimpleJsonObject() throws Exception { + Map map = eval("{\"name\": \"John\", \"age\": 30}"); + + assertEquals("John", map.get("name")); + assertEquals(30, map.get("age")); + assertEquals(2, map.size()); + } + + @Test + public void testInvalidJsonReturnResults() { + Map map = eval("{\"name\": \"John\", \"age\":}"); + + assertEquals("John", map.get("name")); + assertEquals(1, map.size()); + } + + @Test + public void testNonObjectJsonArray() { + Map map = eval("[1, 2, 3]"); + + assertMapListValue(map, "{}", 1, 2, 3); + assertEquals(1, map.size()); + } + + @Test + public void testTopLevelArrayOfObjects() { + Map map = eval("[{\"age\": 1}, {\"age\": 2}]"); + + assertMapListValue(map, "{}.age", 1, 2); + assertEquals(1, map.size()); + } + + @Test + public void testTopLevelArrayOfComplexObjects() { + Map map = + eval("[{\"name\": \"John\", \"age\": 30}, {\"name\": \"Jane\", \"age\": 25}]"); + + assertMapListValue(map, "{}.name", "John", "Jane"); + assertMapListValue(map, "{}.age", 30, 25); + assertEquals(2, map.size()); + } + + @Test + public void testNonObjectJsonPrimitive() { + Object result = JsonExtractAllFunctionImpl.eval("\"just a string\""); + + assertNull(result); + } + + @Test + public void testNonObjectJsonNumber() { + Object result = JsonExtractAllFunctionImpl.eval("42"); + + assertNull(result); + } + + @Test + public void testSingleLevelNesting() { + Map map = eval("{\"user\": {\"name\": \"John\"}, \"system\": \"linux\"}"); + + assertEquals("John", map.get("user.name")); + assertEquals("linux", map.get("system")); + assertEquals(2, map.size()); + } + + @Test + public void testMultiLevelNesting() { + Map map = eval("{\"a\": {\"b\": {\"c\": \"value\"}}}"); + + assertEquals("value", map.get("a.b.c")); + assertEquals(1, map.size()); + } + + @Test + public void testMixedNestedAndFlat() { + Map map = + eval("{\"name\": \"John\", \"address\": {\"city\": \"NYC\", \"zip\": \"10001\"}}"); + + assertEquals("John", map.get("name")); + assertEquals("NYC", map.get("address.city")); + assertEquals("10001", map.get("address.zip")); + assertEquals(3, map.size()); + } + + @Test + public void testDeeplyNestedStructure() { + Map map = + eval("{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"level5\": \"deep\"}}}}}"); + + assertEquals("deep", map.get("level1.level2.level3.level4.level5")); + assertEquals(1, map.size()); + } + + @Test + public void testSimpleArray() { + Map map = eval("{\"tags\": [\"a\", \"b\", \"c\"]}"); + + assertMapListValue(map, "tags{}", "a", "b", "c"); + assertEquals(1, map.size()); + } + + @Test + public void testArrayOfObjects() { + Map map = eval("{\"users\": [{\"name\": \"John\"}, {\"name\": \"Jane\"}]}"); + + assertMapListValue(map, "users{}.name", "John", "Jane"); + assertEquals(1, map.size()); + } + + @Test + public void testNestedArray() { + Map map = eval("{\"data\": {\"items\": [1, 2, 3]}}"); + + assertMapListValue(map, "data.items{}", 1, 2, 3); + assertEquals(1, map.size()); + } + + @Test + public void testNested() { + Map map = + eval( + "{\"data\": {\"items\": [[1, 2, {\"hello\": 3}], 4], \"other\": 5}, \"another\": [6," + + " [7, 8], 9]}"); + + assertMapListValue(map, "data.items{}{}", 1, 2); + assertMapValue(map, "data.items{}{}.hello", 3); + assertMapValue(map, "data.items{}", 4); + assertMapValue(map, "data.other", 5); + assertMapListValue(map, "another{}", 6, 9); + assertMapListValue(map, "another{}{}", 7, 8); + assertEquals(6, map.size()); + } + + @Test + public void testEmptyArray() { + Map map = eval("{\"empty\": []}"); + + Object emptyValue = map.get("empty{}"); + assertNull(emptyValue); + } + + @Test + public void testStringValues() { + Map map = eval("{\"text\": \"hello world\", \"empty\": \"\"}"); + + assertMapValue(map, "text", "hello world"); + assertMapValue(map, "empty", ""); + assertEquals(2, map.size()); + } + + @Test + public void testNumericValues() { + Map map = + eval( + "{\"int\": 42, \"long\": 9223372036854775807, \"hugeNumber\": 9223372036854775808," + + " \"double\": 3.14159}"); + + assertEquals(4, map.size()); + assertEquals(42, map.get("int")); + assertEquals(9223372036854775807L, map.get("long")); + assertEquals(9223372036854775808.0, map.get("hugeNumber")); + assertEquals(3.14159, map.get("double")); + } + + @Test + public void testBooleanValues() { + Map map = eval("{\"isTrue\": true, \"isFalse\": false}"); + + assertEquals(true, map.get("isTrue")); + assertEquals(false, map.get("isFalse")); + assertEquals(2, map.size()); + } + + @Test + public void testNullValues() { + Map map = eval("{\"nullValue\": null, \"notNull\": \"value\"}"); + + assertNull(map.get("nullValue")); + assertEquals("value", map.get("notNull")); + assertEquals(2, map.size()); + } + + @Test + public void testMixedTypesInArray() { + Map map = eval("{\"mixed\": [\"string\", 42, true, null, 3.14]}"); + + List mixed = (List) assertListValue(map, "mixed{}"); + assertEquals(5, mixed.size()); + assertEquals("string", mixed.get(0)); + assertEquals(42, mixed.get(1)); + assertEquals(true, mixed.get(2)); + assertNull(mixed.get(3)); + assertEquals(3.14, mixed.get(4)); + assertEquals(1, map.size()); + } + + @Test + public void testSpecialCharactersInKeys() { + Map map = + eval( + "{\"key.with.dots\": \"value1\", \"key-with-dashes\": \"value2\"," + + " \"key_with_underscores\": \"value3\"}"); + + assertEquals("value1", map.get("key.with.dots")); + assertEquals("value2", map.get("key-with-dashes")); + assertEquals("value3", map.get("key_with_underscores")); + assertEquals(3, map.size()); + } + + @Test + public void testUnicodeCharacters() { + Map map = eval("{\"unicode\": \"こんにちは\", \"emoji\": \"🚀\", \"🚀\": 1}"); + + assertEquals("こんにちは", map.get("unicode")); + assertEquals("🚀", map.get("emoji")); + assertEquals(1, map.get("🚀")); + assertEquals(3, map.size()); + } + + @Test + public void testComplexNestedStructure() { + Map map = + eval( + "{\"user\": {\"profile\": {\"name\": \"John\", \"contacts\": [{\"type\": \"email\"," + + " \"value\": \"john@example.com\"}, {\"type\": \"phone\", \"value\":" + + " \"123-456-7890\"}]}, \"preferences\": {\"theme\": \"dark\", \"notifications\":" + + " true}}}"); + + assertEquals("John", map.get("user.profile.name")); + assertMapListValue(map, "user.profile.contacts{}.type", "email", "phone"); + assertMapListValue(map, "user.profile.contacts{}.value", "john@example.com", "123-456-7890"); + assertEquals("dark", map.get("user.preferences.theme")); + assertEquals(true, map.get("user.preferences.notifications")); + assertEquals(5, map.size()); + } + + @Test + public void testLargeJsonObject() { + StringBuilder jsonBuilder = new StringBuilder("{"); + for (int i = 0; i < 100; i++) { + if (i > 0) jsonBuilder.append(","); + jsonBuilder.append("\"field").append(i).append("\": ").append(i); + } + jsonBuilder.append("}"); + + Map map = eval(jsonBuilder.toString()); + assertEquals(100, map.size()); + assertEquals(0, map.get("field0")); + assertEquals(99, map.get("field99")); + } +} diff --git a/core/src/test/java/org/opensearch/sql/expression/parse/RegexCommonUtilsTest.java b/core/src/test/java/org/opensearch/sql/expression/parse/RegexCommonUtilsTest.java index 5e0f58e3b95..2503b3929f1 100644 --- a/core/src/test/java/org/opensearch/sql/expression/parse/RegexCommonUtilsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/parse/RegexCommonUtilsTest.java @@ -194,12 +194,15 @@ public void testGetNamedGroupCandidatesWithNumericNames() { } @Test - public void testGetNamedGroupCandidatesSpecialCharacters() { - // Test that groups with special characters are not captured (only alphanumeric starting with - // letter) - String pattern = "(?[a-z]+)\\s+(?<123invalid>[0-9]+)\\s+(?.*)"; - List groups = RegexCommonUtils.getNamedGroupCandidates(pattern); - assertEquals(0, groups.size()); + public void testGetNamedGroupCandidatesWithInvalidCharactersThrowsException() { + // Test that groups with invalid characters throw exception (even if some are valid) + String pattern = "(?[a-z]+)\\s+(?<123invalid>[0-9]+)\\s+(?.*)"; + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> RegexCommonUtils.getNamedGroupCandidates(pattern)); + // Should fail on the first invalid group name found + assertTrue(exception.getMessage().contains("Invalid capture group name")); } @Test @@ -211,4 +214,100 @@ public void testGetNamedGroupCandidatesValidAlphanumeric() { assertEquals("groupA", groups.get(0)); assertEquals("group2B", groups.get(1)); } + + @Test + public void testGetNamedGroupCandidatesWithUnderscore() { + // Test that underscores in named groups throw IllegalArgumentException + String patternWithUnderscore = ".+@(?.+)"; + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> RegexCommonUtils.getNamedGroupCandidates(patternWithUnderscore)); + assertTrue(exception.getMessage().contains("Invalid capture group name 'domain_name'")); + assertTrue( + exception + .getMessage() + .contains("must start with a letter and contain only letters and digits")); + } + + @Test + public void testGetNamedGroupCandidatesWithHyphen() { + // Test that hyphens in named groups throw IllegalArgumentException + String patternWithHyphen = ".+@(?.+)"; + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> RegexCommonUtils.getNamedGroupCandidates(patternWithHyphen)); + assertTrue(exception.getMessage().contains("Invalid capture group name 'domain-name'")); + assertTrue( + exception + .getMessage() + .contains("must start with a letter and contain only letters and digits")); + } + + @Test + public void testGetNamedGroupCandidatesWithDot() { + // Test that dots in named groups throw IllegalArgumentException + String patternWithDot = ".+@(?.+)"; + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> RegexCommonUtils.getNamedGroupCandidates(patternWithDot)); + assertTrue(exception.getMessage().contains("Invalid capture group name 'domain.name'")); + } + + @Test + public void testGetNamedGroupCandidatesWithSpace() { + // Test that spaces in named groups throw IllegalArgumentException + String patternWithSpace = ".+@(?.+)"; + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> RegexCommonUtils.getNamedGroupCandidates(patternWithSpace)); + assertTrue(exception.getMessage().contains("Invalid capture group name 'domain name'")); + } + + @Test + public void testGetNamedGroupCandidatesStartingWithDigit() { + // Test that group names starting with digit throw IllegalArgumentException + String patternStartingWithDigit = ".+@(?<1domain>.+)"; + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> RegexCommonUtils.getNamedGroupCandidates(patternStartingWithDigit)); + assertTrue(exception.getMessage().contains("Invalid capture group name '1domain'")); + } + + @Test + public void testGetNamedGroupCandidatesWithSpecialCharacters() { + // Test that special characters in named groups throw IllegalArgumentException + String patternWithSpecialChar = ".+@(?.+)"; + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> RegexCommonUtils.getNamedGroupCandidates(patternWithSpecialChar)); + assertTrue(exception.getMessage().contains("Invalid capture group name 'domain@name'")); + } + + @Test + public void testGetNamedGroupCandidatesWithValidCamelCase() { + // Test that valid camelCase group names work correctly + String validPattern = "(?\\w+)@(?\\w+)"; + List groups = RegexCommonUtils.getNamedGroupCandidates(validPattern); + assertEquals(2, groups.size()); + assertEquals("userName", groups.get(0)); + assertEquals("domainName", groups.get(1)); + } + + @Test + public void testGetNamedGroupCandidatesWithMixedInvalidValid() { + // Test that even one invalid group name fails the entire validation + String patternWithMixed = + "(?[a-z]+)\\s+(?[0-9]+)\\s+(?.*)"; + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> RegexCommonUtils.getNamedGroupCandidates(patternWithMixed)); + assertTrue(exception.getMessage().contains("Invalid capture group name 'invalid_name'")); + } } diff --git a/core/src/test/java/org/opensearch/sql/utils/ComparableLinkedHashMapTest.java b/core/src/test/java/org/opensearch/sql/utils/ComparableLinkedHashMapTest.java new file mode 100644 index 00000000000..5cf5cad6bd1 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/utils/ComparableLinkedHashMapTest.java @@ -0,0 +1,269 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Comparator; +import java.util.Iterator; +import java.util.TreeSet; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.data.utils.ComparableLinkedHashMap; + +public class ComparableLinkedHashMapTest { + + @Test + public void testEmptyMaps() { + ComparableLinkedHashMap map1 = new ComparableLinkedHashMap<>(); + ComparableLinkedHashMap map2 = new ComparableLinkedHashMap<>(); + + assertEquals(0, map1.compareTo(map2)); + } + + @Test + public void testOneEmptyMap() { + ComparableLinkedHashMap map1 = new ComparableLinkedHashMap<>(); + ComparableLinkedHashMap map2 = new ComparableLinkedHashMap<>(); + map2.put("a", 1); + + assertTrue(map1.compareTo(map2) < 0); + assertTrue(map2.compareTo(map1) > 0); + } + + @Test + public void testDifferentKeys() { + ComparableLinkedHashMap map1 = new ComparableLinkedHashMap<>(); + map1.put("a", 1); + + ComparableLinkedHashMap map2 = new ComparableLinkedHashMap<>(); + map2.put("b", 1); + + assertTrue(map1.compareTo(map2) < 0); + } + + @Test + public void testEqualMaps() { + ComparableLinkedHashMap map1 = new ComparableLinkedHashMap<>(); + map1.put("a", 1); + map1.put("b", 2); + + ComparableLinkedHashMap map2 = new ComparableLinkedHashMap<>(); + map2.put("a", 1); + map2.put("b", 2); + + assertEquals(0, map1.compareTo(map2)); + } + + @Test + public void testDifferentFirstValue() { + ComparableLinkedHashMap map1 = new ComparableLinkedHashMap<>(); + map1.put("a", 1); + map1.put("b", 2); + + ComparableLinkedHashMap map2 = new ComparableLinkedHashMap<>(); + map2.put("a", 3); + map2.put("b", 2); + + assertTrue(map1.compareTo(map2) < 0); + assertTrue(map2.compareTo(map1) > 0); + } + + @Test + public void testDifferentLaterValue() { + ComparableLinkedHashMap map1 = new ComparableLinkedHashMap<>(); + map1.put("a", 1); + map1.put("b", 2); + map1.put("c", 3); + + ComparableLinkedHashMap map2 = new ComparableLinkedHashMap<>(); + map2.put("a", 1); + map2.put("b", 3); + map2.put("c", 3); + + assertTrue(map1.compareTo(map2) < 0); + assertTrue(map2.compareTo(map1) > 0); + } + + @Test + public void testDifferentSizes() { + ComparableLinkedHashMap map1 = new ComparableLinkedHashMap<>(); + map1.put("a", 1); + map1.put("b", 2); + + ComparableLinkedHashMap map2 = new ComparableLinkedHashMap<>(); + map2.put("a", 1); + + assertTrue(map1.compareTo(map2) > 0); + assertTrue(map2.compareTo(map1) < 0); + } + + @Test + public void testNullValues() { + ComparableLinkedHashMap map1 = new ComparableLinkedHashMap<>(); + map1.put("a", null); + map1.put("b", 2); + + ComparableLinkedHashMap map2 = new ComparableLinkedHashMap<>(); + map2.put("a", 1); + map2.put("b", 2); + + assertTrue(map1.compareTo(map2) < 0); + assertTrue(map2.compareTo(map1) > 0); + + ComparableLinkedHashMap map3 = new ComparableLinkedHashMap<>(); + map3.put("a", null); + map3.put("b", 2); + + assertEquals(0, map1.compareTo(map3)); + } + + @Test + public void testCustomObjects() { + class Person { + String name; + + Person(String name) { + this.name = name; + } + + @Override + public String toString() { + return name; + } + } + + ComparableLinkedHashMap map1 = new ComparableLinkedHashMap<>(); + map1.put("a", new Person("Alice")); + map1.put("b", new Person("Bob")); + + ComparableLinkedHashMap map2 = new ComparableLinkedHashMap<>(); + map2.put("a", new Person("Alice")); + map2.put("b", new Person("Charlie")); + + assertTrue(map1.compareTo(map2) < 0); + assertTrue(map2.compareTo(map1) > 0); + } + + @Test + public void testMixedTypes() { + ComparableLinkedHashMap map1 = new ComparableLinkedHashMap<>(); + map1.put("a", 1); + map1.put("b", "test"); + + ComparableLinkedHashMap map2 = new ComparableLinkedHashMap<>(); + map2.put("a", 1); + map2.put("b", "test"); + + assertEquals(0, map1.compareTo(map2)); + + ComparableLinkedHashMap map3 = new ComparableLinkedHashMap<>(); + map3.put("a", 1); + map3.put("b", "test2"); + + assertTrue(map1.compareTo(map3) < 0); + assertTrue(map3.compareTo(map1) > 0); + } + + @Test + public void testWithTreeSet() { + TreeSet> set = new TreeSet<>(); + + ComparableLinkedHashMap map1 = new ComparableLinkedHashMap<>(); + map1.put("a", 1); + map1.put("b", 2); + + ComparableLinkedHashMap map2 = new ComparableLinkedHashMap<>(); + map2.put("a", 1); + map2.put("b", 3); + + ComparableLinkedHashMap map3 = new ComparableLinkedHashMap<>(); + map3.put("a", 0); + map3.put("b", 4); + + set.add(map2); + set.add(map1); + set.add(map3); + + Iterator> iterator = set.iterator(); + ComparableLinkedHashMap first = iterator.next(); + ComparableLinkedHashMap second = iterator.next(); + ComparableLinkedHashMap third = iterator.next(); + + assertEquals(Integer.valueOf(0), first.get("a")); + assertEquals(Integer.valueOf(4), first.get("b")); + + assertEquals(Integer.valueOf(1), second.get("a")); + assertEquals(Integer.valueOf(2), second.get("b")); + + assertEquals(Integer.valueOf(1), third.get("a")); + assertEquals(Integer.valueOf(3), third.get("b")); + + assertEquals(3, set.size()); + } + + @Test + public void testWithComparator() { + Comparator> comparator = + ComparableLinkedHashMap::compareTo; + + ComparableLinkedHashMap map1 = new ComparableLinkedHashMap<>(); + map1.put("a", 5); + + ComparableLinkedHashMap map2 = new ComparableLinkedHashMap<>(); + map2.put("a", 3); + + assertTrue(comparator.compare(map1, map2) > 0); + assertTrue(comparator.compare(map2, map1) < 0); + } + + @Test + public void testEqualValuesDifferentKeys() { + ComparableLinkedHashMap map1 = new ComparableLinkedHashMap<>(); + map1.put("a", 1); + map1.put("b", 2); + map1.put("c", 3); + + ComparableLinkedHashMap map2 = new ComparableLinkedHashMap<>(); + map2.put("d", 1); + map2.put("e", 2); + map2.put("f", 3); + + assertTrue(map1.compareTo(map2) < 0); + } + + @Test + public void testDifferentOrder() { + ComparableLinkedHashMap map1 = new ComparableLinkedHashMap<>(); + map1.put("a", 1); + map1.put("b", 2); + + ComparableLinkedHashMap map2 = new ComparableLinkedHashMap<>(); + map2.put("a", 2); + map2.put("b", 1); + + assertTrue(map1.compareTo(map2) < 0); + assertTrue(map2.compareTo(map1) > 0); + } + + @Test + public void testLargeMaps() { + ComparableLinkedHashMap map1 = new ComparableLinkedHashMap<>(); + for (int i = 0; i < 100; i++) { + map1.put("key" + i, i); + } + + ComparableLinkedHashMap map2 = new ComparableLinkedHashMap<>(); + for (int i = 0; i < 100; i++) { + map2.put("key" + i, i); + } + + assertEquals(0, map1.compareTo(map2)); + + map2.put("key", 100); + assertTrue(map1.compareTo(map2) < 0); + } +} diff --git a/docs/category.json b/docs/category.json index cb40f1ebbcd..7ebe643373b 100644 --- a/docs/category.json +++ b/docs/category.json @@ -1,12 +1,11 @@ { "bash": [ - "user/ppl/interfaces/endpoint.rst", - "user/ppl/interfaces/protocol.rst", - "user/ppl/admin/settings.rst", "user/optimization/optimization.rst", "user/admin/settings.rst" ], - "ppl_cli": [ + "bash_calcite": [ + "user/ppl/interfaces/endpoint.rst", + "user/ppl/interfaces/protocol.rst" ], "sql_cli": [ "user/dql/expressions.rst", @@ -41,11 +40,14 @@ "user/ppl/cmd/rare.rst", "user/ppl/cmd/regex.rst", "user/ppl/cmd/rename.rst", + "user/ppl/cmd/multisearch.rst", + "user/ppl/cmd/replace.rst", "user/ppl/cmd/rex.rst", "user/ppl/cmd/search.rst", "user/ppl/cmd/showdatasources.rst", "user/ppl/cmd/sort.rst", "user/ppl/cmd/stats.rst", + "user/ppl/cmd/streamstats.rst", "user/ppl/cmd/subquery.rst", "user/ppl/cmd/syntax.rst", "user/ppl/cmd/timechart.rst", @@ -54,6 +56,7 @@ "user/ppl/cmd/top.rst", "user/ppl/cmd/trendline.rst", "user/ppl/cmd/where.rst", + "user/ppl/functions/collection.rst", "user/ppl/functions/condition.rst", "user/ppl/functions/datetime.rst", "user/ppl/functions/expressions.rst", @@ -62,7 +65,11 @@ "user/ppl/functions/math.rst", "user/ppl/functions/relevance.rst", "user/ppl/functions/string.rst", + "user/ppl/functions/conversion.rst", "user/ppl/general/datatypes.rst", "user/ppl/general/identifiers.rst" + ], + "bash_settings": [ + "user/ppl/admin/settings.rst" ] } diff --git a/docs/dev/testing-doctest.md b/docs/dev/testing-doctest.md index a90a5a2537a..1a966ba50c3 100644 --- a/docs/dev/testing-doctest.md +++ b/docs/dev/testing-doctest.md @@ -52,7 +52,7 @@ For actually testing the code, the goal is to thoroughly test every case, rather ## 1.4 How to use doctest? ### 1.4.2 How to run existing doctest? -Doctest runs with project build by `./gradlew build`. You can also only run doctest by `./gradlew doctest` +Doctest runs with project build by `./gradlew build`. You can also only run doctest by `./gradlew doctest`. If a Prometheus instance isn't available locally, add `-DignorePrometheus` (or set the property to any value other than `false`) to skip Prometheus setup and the Prometheus-specific doctest scenarios. Make sure you don't have any OpenSearch instance running at `http://localhost:9200` @@ -60,7 +60,7 @@ Make sure you don't have any OpenSearch instance running at `http://localhost:92 1. If you want to add a new doc, you can add it to `docs` folder, under correct sub-folder, in `.rst` format. > **Attention**: For code examples in documentation, a Mixing usage of `cli` and `bash` in one doc is not supported yet. 2. Add your new doc file path to `docs/category.json` by its category -3. Run doctest `./gradlew doctest` to see if your tests can pass +3. Run doctest `./gradlew doctest` (optionally with `-DignorePrometheus`) to see if your tests can pass Currently, there is a `sample` folder under `docs` module to help you get started. diff --git a/docs/user/admin/settings.rst b/docs/user/admin/settings.rst index 61fc9fa83d9..cd8ee2458ae 100644 --- a/docs/user/admin/settings.rst +++ b/docs/user/admin/settings.rst @@ -161,7 +161,7 @@ Result set:: } plugins.query.size_limit -=========================== +======================== Description ----------- @@ -188,6 +188,43 @@ Result set:: } } +plugins.query.buckets +===================== + +Version +------- +3.4.0 + +Description +----------- + +This configuration indicates how many aggregation buckets will return in a single response. The default value equals to ``plugins.query.size_limit``. +You can change the value to any value not greater than the maximum number of aggregation buckets allowed in a single response (`search.max_buckets`), here is an example:: + + >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_query/settings -d '{ + "transient" : { + "plugins.query.buckets" : 1000 + } + }' + +Result set:: + + { + "acknowledged" : true, + "persistent" : { }, + "transient" : { + "plugins" : { + "query" : { + "buckets" : "1000" + } + } + } + } + +Limitations +----------- +The number of aggregation buckets is fixed to ``1000`` in v2. ``plugins.query.buckets`` can only effect the number of aggregation buckets when calcite enabled. + plugins.query.memory_limit ========================== diff --git a/docs/user/dql/metadata.rst b/docs/user/dql/metadata.rst index 8135e3684d4..3b277cd978f 100644 --- a/docs/user/dql/metadata.rst +++ b/docs/user/dql/metadata.rst @@ -35,7 +35,7 @@ Example 1: Show All Indices Information SQL query:: os> SHOW TABLES LIKE '%' - fetched rows / total rows = 18/18 + fetched rows / total rows = 20/20 +----------------+-------------+------------------+------------+---------+----------+------------+-----------+---------------------------+----------------+ | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | TABLE_TYPE | REMARKS | TYPE_CAT | TYPE_SCHEM | TYPE_NAME | SELF_REFERENCING_COL_NAME | REF_GENERATION | |----------------+-------------+------------------+------------+---------+----------+------------+-----------+---------------------------+----------------| @@ -52,6 +52,8 @@ SQL query:: | docTestCluster | null | otellogs | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | people | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | state_country | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | time_data | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | time_data2 | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | time_test | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | weblogs | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | wildcard | BASE TABLE | null | null | null | null | null | null | diff --git a/docs/user/ppl/admin/datasources.rst b/docs/user/ppl/admin/datasources.rst index 0b423b350bb..c5f9adfd85a 100644 --- a/docs/user/ppl/admin/datasources.rst +++ b/docs/user/ppl/admin/datasources.rst @@ -261,6 +261,28 @@ PPL query for searching PROMETHEUS TABLES:: +---------------+--------------+--------------------------------------------+------------+------+----------------------------------------------------+ +.. _datasources-prometheus-metadata: + +Fetch metadata for table in Prometheus datasource +================================================= + +After a Prometheus datasource is configured, you can inspect the schema of any metric by running the ``describe`` command against the fully qualified table name. For example:: + +PPL query:: + + PPL> describe my_prometheus.prometheus_http_requests_total; + fetched rows / total rows = 6/6 + +---------------+--------------+--------------------------------+-------------+-----------+ + | TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | COLUMN_NAME | DATA_TYPE | + |---------------+--------------+--------------------------------+-------------+-----------| + | my_prometheus | default | prometheus_http_requests_total | handler | string | + | my_prometheus | default | prometheus_http_requests_total | code | string | + | my_prometheus | default | prometheus_http_requests_total | instance | string | + | my_prometheus | default | prometheus_http_requests_total | @timestamp | timestamp | + | my_prometheus | default | prometheus_http_requests_total | @value | double | + | my_prometheus | default | prometheus_http_requests_total | job | string | + +---------------+--------------+--------------------------------+-------------+-----------+ + Limitations =========== diff --git a/docs/user/ppl/admin/settings.rst b/docs/user/ppl/admin/settings.rst index 389a5c24be8..d99cdc6c2d0 100644 --- a/docs/user/ppl/admin/settings.rst +++ b/docs/user/ppl/admin/settings.rst @@ -73,22 +73,6 @@ PPL query:: "status": 400 } -Example 3 ---------- - -You can reset the setting to default value like this. - -PPL query:: - - sh$ curl -sS -H 'Content-Type: application/json' \ - ... -X PUT localhost:9200/_plugins/_query/settings \ - ... -d '{"transient" : {"plugins.ppl.enabled" : null}}' - { - "acknowledged": true, - "persistent": {}, - "transient": {} - } - plugins.query.memory_limit ========================== @@ -147,18 +131,44 @@ Change the size_limit to 1000:: "transient": {} } -Rollback to default value:: +Note: the legacy settings of ``opendistro.query.size_limit`` is deprecated, it will fallback to the new settings if you request an update with the legacy name. + +plugins.query.buckets +===================== + +Version +------- +3.4.0 + +Description +----------- + +This configuration indicates how many aggregation buckets will return in a single response. The default value equals to ``plugins.query.size_limit``. +You can change the value to any value not greater than the maximum number of aggregation buckets allowed in a single response (`search.max_buckets`), here is an example:: + + >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_query/settings -d '{ + "transient" : { + "plugins.query.buckets" : 1000 + } + }' + +Result set:: - sh$ curl -sS -H 'Content-Type: application/json' \ - ... -X PUT localhost:9200/_plugins/_query/settings \ - ... -d '{"persistent" : {"plugins.query.size_limit" : null}}' { - "acknowledged": true, - "persistent": {}, - "transient": {} + "acknowledged" : true, + "persistent" : { }, + "transient" : { + "plugins" : { + "query" : { + "buckets" : "1000" + } + } + } } -Note: the legacy settings of ``opendistro.query.size_limit`` is deprecated, it will fallback to the new settings if you request an update with the legacy name. +Limitations +----------- +The number of aggregation buckets is fixed to ``1000`` in v2. ``plugins.query.buckets`` can only effect the number of aggregation buckets when calcite enabled. plugins.calcite.all_join_types.allowed ====================================== @@ -200,8 +210,10 @@ This configuration is introduced since 3.3.0 which is used to switch some behavi The behaviours it controlled includes: - The default value of argument ``bucket_nullable`` in ``stats`` command. Check `stats command <../cmd/stats.rst>`_ for details. +- The return value of ``divide`` and ``/`` operator. Check `expressions <../functions/expressions.rst>`_ for details. +- The default value of argument ``usenull`` in ``top`` and ``rare`` commands. Check `top command <../cmd/top.rst>`_ and `rare command <../cmd/rare.rst>`_ for details. -Example +Example 1 ------- You can update the setting with a new value like this. @@ -227,6 +239,22 @@ PPL query:: } } +Example 2 +--------- + +Reset to default (true) by setting to null: + +PPL query:: + + sh$ curl -sS -H 'Content-Type: application/json' \ + ... -X PUT localhost:9200/_plugins/_query/settings \ + ... -d '{"transient" : {"plugins.ppl.syntax.legacy.preferred" : null}}' + { + "acknowledged": true, + "persistent": {}, + "transient": {} + } + plugins.ppl.values.max.limit ============================ @@ -270,41 +298,94 @@ PPL query:: Example 2 --------- -Reset to default (unlimited) by setting to null: +Set to 0 explicitly for unlimited values: PPL query:: sh$ curl -sS -H 'Content-Type: application/json' \ ... -X PUT localhost:9200/_plugins/_query/settings \ - ... -d '{"transient" : {"plugins.ppl.values.max.limit" : null}}' + ... -d '{"transient" : {"plugins.ppl.values.max.limit" : "0"}}' { "acknowledged": true, "persistent": {}, + "transient": { + "plugins": { + "ppl": { + "values": { + "max": { + "limit": "0" + } + } + } + } + } + } + + +plugins.ppl.subsearch.maxout +============================ + +Description +----------- + +The size configures the maximum of rows to return from subsearch. The default value is: ``10000``. A value of ``0`` indicates that the restriction is unlimited. + +Version +------- +3.4.0 + +Example +------- + +Change the subsearch.maxout to unlimited:: + + sh$ curl -sS -H 'Content-Type: application/json' \ + ... -X PUT localhost:9200/_plugins/_query/settings \ + ... -d '{"persistent" : {"plugins.ppl.subsearch.maxout" : "0"}}' + { + "acknowledged": true, + "persistent": { + "plugins": { + "ppl": { + "subsearch": { + "maxout": "0" + } + } + } + }, "transient": {} } -Example 3 ---------- +plugins.ppl.join.subsearch_maxout +================================= -Set to 0 explicitly for unlimited values: +Description +----------- -PPL query:: +The size configures the maximum of rows from subsearch to join against. This configuration impacts ``join`` command. The default value is: ``50000``. A value of ``0`` indicates that the restriction is unlimited. + +Version +------- +3.4.0 + +Example +------- + +Change the join.subsearch_maxout to 5000:: sh$ curl -sS -H 'Content-Type: application/json' \ ... -X PUT localhost:9200/_plugins/_query/settings \ - ... -d '{"transient" : {"plugins.ppl.values.max.limit" : "0"}}' + ... -d '{"persistent" : {"plugins.ppl.join.subsearch_maxout" : "5000"}}' { "acknowledged": true, - "persistent": {}, - "transient": { + "persistent": { "plugins": { "ppl": { - "values": { - "max": { - "limit": "0" - } + "join": { + "subsearch_maxout": "5000" } } } - } + }, + "transient": {} } diff --git a/docs/user/ppl/cmd/append.rst b/docs/user/ppl/cmd/append.rst index 982e0e33024..25303aeb87b 100644 --- a/docs/user/ppl/cmd/append.rst +++ b/docs/user/ppl/cmd/append.rst @@ -24,6 +24,11 @@ append * sub-search: mandatory. Executes PPL commands as a secondary search. +Limitations +=========== + +* **Schema Compatibility**: When fields with the same name exist between the main search and sub-search but have incompatible types, the query will fail with an error. To avoid type conflicts, ensure that fields with the same name have the same data type, or use different field names (e.g., by renaming with ``eval`` or using ``fields`` to select non-conflicting columns). + Example 1: Append rows from a count aggregation to existing search result =============================================================== @@ -64,23 +69,3 @@ PPL query:: | 101 | M | null | +-----+--------+-------+ -Example 3: Append rows with column type conflict -============================================= - -This example shows how column type conflicts are handled when appending results. Same name columns with different types will generate two different columns in appended result. - -PPL query:: - - os> source=accounts | stats sum(age) as sum by gender, state | sort -sum | head 5 | append [ source=accounts | stats sum(age) as sum by gender | eval sum = cast(sum as double) ]; - fetched rows / total rows = 6/6 - +------+--------+-------+-------+ - | sum | gender | state | sum0 | - |------+--------+-------+-------| - | 36 | M | TN | null | - | 33 | M | MD | null | - | 32 | M | IL | null | - | 28 | F | VA | null | - | null | F | null | 28.0 | - | null | M | null | 101.0 | - +------+--------+-------+-------+ - diff --git a/docs/user/ppl/cmd/describe.rst b/docs/user/ppl/cmd/describe.rst index 865e6cc8f93..c732480e328 100644 --- a/docs/user/ppl/cmd/describe.rst +++ b/docs/user/ppl/cmd/describe.rst @@ -67,22 +67,7 @@ PPL query:: +----------------+ -Example 3: Fetch metadata for table in prometheus dataSource -========================================================= +Example 3: Fetch metadata for table in Prometheus datasource +============================================================ -The example retrieves table info for ``prometheus_http_requests_total`` metric in prometheus dataSource. - -PPL query:: - - os> describe my_prometheus.prometheus_http_requests_total; - fetched rows / total rows = 6/6 - +---------------+--------------+--------------------------------+-------------+-----------+ - | TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | COLUMN_NAME | DATA_TYPE | - |---------------+--------------+--------------------------------+-------------+-----------| - | my_prometheus | default | prometheus_http_requests_total | handler | string | - | my_prometheus | default | prometheus_http_requests_total | code | string | - | my_prometheus | default | prometheus_http_requests_total | instance | string | - | my_prometheus | default | prometheus_http_requests_total | @timestamp | timestamp | - | my_prometheus | default | prometheus_http_requests_total | @value | double | - | my_prometheus | default | prometheus_http_requests_total | job | string | - +---------------+--------------+--------------------------------+-------------+-----------+ +See `Fetch metadata for table in Prometheus datasource <../admin/datasources.rst>`_ for more context. diff --git a/docs/user/ppl/cmd/eventstats.rst b/docs/user/ppl/cmd/eventstats.rst index f81401934ec..958b28e606b 100644 --- a/docs/user/ppl/cmd/eventstats.rst +++ b/docs/user/ppl/cmd/eventstats.rst @@ -1,5 +1,5 @@ ============= -evenstats +eventstats ============= .. rubric:: Table of contents @@ -13,7 +13,7 @@ Description ============ | (Experimental) | (From 3.1.0) -| Using ``evenstats`` command to enriches your event data with calculated summary statistics. It operates by analyzing specified fields within your events, computing various statistical measures, and then appending these results as new fields to each original event. +| Using ``eventstats`` command to enriches your event data with calculated summary statistics. It operates by analyzing specified fields within your events, computing various statistical measures, and then appending these results as new fields to each original event. | Key aspects of `eventstats`: diff --git a/docs/user/ppl/cmd/fillnull.rst b/docs/user/ppl/cmd/fillnull.rst index 6d05581fd93..483755f723f 100644 --- a/docs/user/ppl/cmd/fillnull.rst +++ b/docs/user/ppl/cmd/fillnull.rst @@ -16,15 +16,33 @@ Using ``fillnull`` command to fill null with provided value in one or more field Syntax ============ + fillnull with [in ] fillnull using = [, = ] -* replacement: mandatory. The value used to replace `null`s. -* field-list: optional. The comma-delimited field list. The `null` values in the field will be replaced with the values from the replacement. From 3.1.0, when ``plugins.calcite.enabled`` is true, if no field specified, the replacement is applied to all fields. +fillnull value= [] + + +Parameters +============ + +* replacement: Mandatory. The value used to replace `null`s. + +* field-list: Optional. Comma-delimited (when using ``with`` or ``using``) or space-delimited (when using ``value=``) list of fields. The `null` values in the field will be replaced with the values from the replacement. **Default:** If no field specified, the replacement is applied to all fields. + +**Syntax Variations:** + +* ``with in `` - Apply same value to specified fields +* ``using =, ...`` - Apply different values to different fields +* ``value= []`` - Alternative syntax with optional space-delimited field list + + +Examples +============ -Example 1: replace null values with a specified value on one field -================================================================== +Example 1: Replace null values with a specified value on one field +------------------------------------------------------------------- PPL query:: @@ -39,8 +57,8 @@ PPL query:: | daleadams@boink.com | null | +-----------------------+----------+ -Example 2: replace null values with a specified value on multiple fields -======================================================================== +Example 2: Replace null values with a specified value on multiple fields +------------------------------------------------------------------------- PPL query:: @@ -55,10 +73,8 @@ PPL query:: | daleadams@boink.com | | +-----------------------+-------------+ -Example 3: replace null values with a specified value on all fields -=================================================================== - -This example only works when Calcite enabled. +Example 3: Replace null values with a specified value on all fields +-------------------------------------------------------------------- PPL query:: @@ -73,8 +89,8 @@ PPL query:: | daleadams@boink.com | | +-----------------------+-------------+ -Example 4: replace null values with multiple specified values on multiple fields -================================================================================ +Example 4: Replace null values with multiple specified values on multiple fields +--------------------------------------------------------------------------------- PPL query:: @@ -90,7 +106,51 @@ PPL query:: +-----------------------+---------------+ +Example 5: Replace null with specified value on specific fields (value= syntax) +-------------------------------------------------------------------------------- + +PPL query:: + + os> source=accounts | fields email, employer | fillnull value="" email employer; + fetched rows / total rows = 4/4 + +-----------------------+-------------+ + | email | employer | + |-----------------------+-------------| + | amberduke@pyrami.com | Pyrami | + | hattiebond@netagy.com | Netagy | + | | Quility | + | daleadams@boink.com | | + +-----------------------+-------------+ + +Example 6: Replace null with specified value on all fields (value= syntax) +--------------------------------------------------------------------------- + +When no field list is specified, the replacement applies to all fields in the result. + +PPL query:: + + os> source=accounts | fields email, employer | fillnull value=''; + fetched rows / total rows = 4/4 + +-----------------------+-------------+ + | email | employer | + |-----------------------+-------------| + | amberduke@pyrami.com | Pyrami | + | hattiebond@netagy.com | Netagy | + | | Quility | + | daleadams@boink.com | | + +-----------------------+-------------+ + Limitations -=========== +============ * The ``fillnull`` command is not rewritten to OpenSearch DSL, it is only executed on the coordination node. -* Before 3.1.0, at least one field is required. +* When applying the same value to all fields without specifying field names, all fields must be the same type. For mixed types, use separate fillnull commands or explicitly specify fields. +* The replacement value type must match ALL field types in the field list. When applying the same value to multiple fields, all fields must be the same type (all strings or all numeric). + + **Example:** + + .. code-block:: sql + + # This FAILS - same value for mixed-type fields + source=accounts | fillnull value=0 firstname, age + # ERROR: fillnull failed: replacement value type INTEGER is not compatible with field 'firstname' (type: VARCHAR). The replacement value type must match the field type. + diff --git a/docs/user/ppl/cmd/join.rst b/docs/user/ppl/cmd/join.rst index 92244f97f69..3b986071261 100644 --- a/docs/user/ppl/cmd/join.rst +++ b/docs/user/ppl/cmd/join.rst @@ -39,6 +39,10 @@ Extended syntax since 3.3.0 Configuration ============= + +plugins.calcite.enabled +----------------------- + This command requires Calcite enabled. In 3.0.0, as an experimental the Calcite configuration is disabled by default. Enable Calcite:: @@ -63,6 +67,32 @@ Result set:: "transient": {} } + +plugins.ppl.join.subsearch_maxout +--------------------------------- + +The size configures the maximum of rows from subsearch to join against. The default value is: ``50000``. A value of ``0`` indicates that the restriction is unlimited. + +Change the join.subsearch_maxout to 5000:: + + sh$ curl -sS -H 'Content-Type: application/json' \ + ... -X PUT localhost:9200/_plugins/_query/settings \ + ... -d '{"persistent" : {"plugins.ppl.join.subsearch_maxout" : "5000"}}' + { + "acknowledged": true, + "persistent": { + "plugins": { + "ppl": { + "join": { + "subsearch_maxout": "5000" + } + } + } + }, + "transient": {} + } + + Usage ===== diff --git a/docs/user/ppl/cmd/multisearch.rst b/docs/user/ppl/cmd/multisearch.rst new file mode 100644 index 00000000000..2bac577ef23 --- /dev/null +++ b/docs/user/ppl/cmd/multisearch.rst @@ -0,0 +1,141 @@ +============= +multisearch +============= + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + + +Description +============ +| (Experimental) +| Using ``multisearch`` command to run multiple search subsearches and merge their results together. The command allows you to combine data from different queries on the same or different sources, and optionally apply subsequent processing to the combined result set. + +| Key aspects of ``multisearch``: + +1. Combines results from multiple search operations into a single result set. +2. Each subsearch can have different filtering criteria, data transformations, and field selections. +3. Results are merged and can be further processed with aggregations, sorting, and other PPL commands. +4. Particularly useful for comparative analysis, union operations, and creating comprehensive datasets from multiple search criteria. +5. Supports timestamp-based result interleaving when working with time-series data. + +| Use Cases: + +* **Comparative Analysis**: Compare metrics across different segments, regions, or time periods +* **Success Rate Monitoring**: Calculate success rates by comparing successful vs. total operations +* **Multi-source Data Combination**: Merge data from different indices or apply different filters to the same source +* **A/B Testing Analysis**: Combine results from different test groups for comparison +* **Time-series Data Merging**: Interleave events from multiple sources based on timestamps + +Syntax +====== +| multisearch ... + +**Requirements:** + +* **Minimum 2 subsearches required** - multisearch must contain at least two subsearch blocks +* **Maximum unlimited** - you can specify as many subsearches as needed + +**Subsearch Format:** + +* Each subsearch must be enclosed in square brackets: ``[search ...]`` +* Each subsearch must start with the ``search`` keyword +* Syntax: ``[search source=index | commands...]`` +* Description: Each subsearch is a complete search pipeline enclosed in square brackets + * Supported commands in subsearches: All PPL commands are supported (``where``, ``eval``, ``fields``, ``head``, ``rename``, ``stats``, ``sort``, ``dedup``, etc.) + +* result-processing: optional. Commands applied to the merged results. + + * Description: After the multisearch operation, you can apply any PPL command to process the combined results, such as ``stats``, ``sort``, ``head``, etc. + +Limitations +=========== + +* **Minimum Subsearches**: At least two subsearches must be specified +* **Schema Compatibility**: When fields with the same name exist across subsearches but have incompatible types, the query will fail with an error. To avoid type conflicts, ensure that fields with the same name have the same data type across all subsearches, or use different field names (e.g., by renaming with ``eval`` or using ``fields`` to select non-conflicting columns). + +Usage +===== + +Basic multisearch:: + + | multisearch [search source=table | where condition1] [search source=table | where condition2] + | multisearch [search source=index1 | fields field1, field2] [search source=index2 | fields field1, field2] + | multisearch [search source=table | where status="success"] [search source=table | where status="error"] + +Example 1: Basic Age Group Analysis +=================================== + +Combine young and adult customers into a single result set for further analysis. + +PPL query:: + + os> | multisearch [search source=accounts | where age < 30 | eval age_group = "young" | fields firstname, age, age_group] [search source=accounts | where age >= 30 | eval age_group = "adult" | fields firstname, age, age_group] | sort age; + fetched rows / total rows = 4/4 + +-----------+-----+-----------+ + | firstname | age | age_group | + |-----------+-----+-----------| + | Nanette | 28 | young | + | Amber | 32 | adult | + | Dale | 33 | adult | + | Hattie | 36 | adult | + +-----------+-----+-----------+ + +Example 2: Success Rate Pattern +=============================== + +Combine high-balance and all valid accounts for comparison analysis. + +PPL query:: + + os> | multisearch [search source=accounts | where balance > 20000 | eval query_type = "high_balance" | fields firstname, balance, query_type] [search source=accounts | where balance > 0 AND balance <= 20000 | eval query_type = "regular" | fields firstname, balance, query_type] | sort balance desc; + fetched rows / total rows = 4/4 + +-----------+---------+--------------+ + | firstname | balance | query_type | + |-----------+---------+--------------| + | Amber | 39225 | high_balance | + | Nanette | 32838 | high_balance | + | Hattie | 5686 | regular | + | Dale | 4180 | regular | + +-----------+---------+--------------+ + +Example 3: Timestamp Interleaving +================================== + +Combine time-series data from multiple sources with automatic timestamp-based ordering. + +PPL query:: + + os> | multisearch [search source=time_data | where category IN ("A", "B")] [search source=time_data2 | where category IN ("E", "F")] | fields @timestamp, category, value, timestamp | head 5; + fetched rows / total rows = 5/5 + +---------------------+----------+-------+---------------------+ + | @timestamp | category | value | timestamp | + |---------------------+----------+-------+---------------------| + | 2025-08-01 04:00:00 | E | 2001 | 2025-08-01 04:00:00 | + | 2025-08-01 03:47:41 | A | 8762 | 2025-08-01 03:47:41 | + | 2025-08-01 02:30:00 | F | 2002 | 2025-08-01 02:30:00 | + | 2025-08-01 01:14:11 | B | 9015 | 2025-08-01 01:14:11 | + | 2025-08-01 01:00:00 | E | 2003 | 2025-08-01 01:00:00 | + +---------------------+----------+-------+---------------------+ + +Example 4: Type Compatibility - Missing Fields +================================================= + +Demonstrate how missing fields are handled with NULL insertion. + +PPL query:: + + os> | multisearch [search source=accounts | where age < 30 | eval young_flag = "yes" | fields firstname, age, young_flag] [search source=accounts | where age >= 30 | fields firstname, age] | sort age; + fetched rows / total rows = 4/4 + +-----------+-----+------------+ + | firstname | age | young_flag | + |-----------+-----+------------| + | Nanette | 28 | yes | + | Amber | 32 | null | + | Dale | 33 | null | + | Hattie | 36 | null | + +-----------+-----+------------+ + diff --git a/docs/user/ppl/cmd/parse.rst b/docs/user/ppl/cmd/parse.rst index 369fb2d3ece..8e0dc7da080 100644 --- a/docs/user/ppl/cmd/parse.rst +++ b/docs/user/ppl/cmd/parse.rst @@ -114,3 +114,7 @@ There are a few limitations with parse command: For example, the following query will not display the parsed fields ``host`` unless the source field ``email`` is also explicitly included:: source=accounts | parse email '.+@(?.+)' | fields email, host ; + +- Named capture group must start with a letter and contain only letters and digits. + + For detailed Java regex pattern syntax and usage, refer to the `official Java Pattern documentation `_ diff --git a/docs/user/ppl/cmd/rare.rst b/docs/user/ppl/cmd/rare.rst index 8d2011cc1b2..d16dc4878dd 100644 --- a/docs/user/ppl/cmd/rare.rst +++ b/docs/user/ppl/cmd/rare.rst @@ -1,6 +1,6 @@ -============= +==== rare -============= +==== .. rubric:: Table of contents @@ -10,13 +10,13 @@ rare Description -============ +=========== | Using ``rare`` command to find the least common tuple of values of all fields in the field list. **Note**: A maximum of 10 results is returned for each distinct tuple of values of the group-by fields. Syntax -============ +====== rare [by-clause] rare [rare-options] [by-clause] ``(available from 3.1.0+)`` @@ -26,10 +26,13 @@ rare [rare-options] [by-clause] ``(available from 3.1.0+)`` * rare-options: optional. options for the rare command. Supported syntax is [countfield=] [showcount=]. * showcount=: optional. whether to create a field in output that represent a count of the tuple of values. Default value is ``true``. * countfield=: optional. the name of the field that contains count. Default value is ``'count'``. +* usenull=: optional (since 3.4.0). whether to output the null value. The default value of ``usenull`` is determined by ``plugins.ppl.syntax.legacy.preferred``: + * When ``plugins.ppl.syntax.legacy.preferred=true``, ``usenull`` defaults to ``true`` + * When ``plugins.ppl.syntax.legacy.preferred=false``, ``usenull`` defaults to ``false`` Example 1: Find the least common values in a field -=========================================== +================================================== The example finds least common gender of all the accounts. @@ -46,7 +49,7 @@ PPL query:: Example 2: Find the least common values organized by gender -==================================================== +=========================================================== The example finds least common age of all the accounts group by gender. @@ -66,12 +69,10 @@ PPL query:: Example 3: Rare command with Calcite enabled ============================================ -The example finds least common gender of all the accounts when ``plugins.calcite.enabled`` is true. - PPL query:: - PPL> source=accounts | rare gender; - fetched row + os> source=accounts | rare gender; + fetched rows / total rows = 2/2 +--------+-------+ | gender | count | |--------+-------| @@ -83,12 +84,10 @@ PPL query:: Example 4: Specify the count field option ========================================= -The example specifies the count field when ``plugins.calcite.enabled`` is true. - PPL query:: - PPL> source=accounts | rare countfield='cnt' gender; - fetched row + os> source=accounts | rare countfield='cnt' gender; + fetched rows / total rows = 2/2 +--------+-----+ | gender | cnt | |--------+-----| @@ -96,6 +95,36 @@ PPL query:: | M | 3 | +--------+-----+ + +Example 5: Specify the usenull field option +=========================================== + +PPL query:: + + os> source=accounts | rare usenull=false email; + fetched rows / total rows = 3/3 + +-----------------------+-------+ + | email | count | + |-----------------------+-------| + | amberduke@pyrami.com | 1 | + | daleadams@boink.com | 1 | + | hattiebond@netagy.com | 1 | + +-----------------------+-------+ + +PPL query:: + + os> source=accounts | rare usenull=true email; + fetched rows / total rows = 4/4 + +-----------------------+-------+ + | email | count | + |-----------------------+-------| + | null | 1 | + | amberduke@pyrami.com | 1 | + | daleadams@boink.com | 1 | + | hattiebond@netagy.com | 1 | + +-----------------------+-------+ + + Limitations =========== The ``rare`` command is not rewritten to OpenSearch DSL, it is only executed on the coordination node. diff --git a/docs/user/ppl/cmd/replace.rst b/docs/user/ppl/cmd/replace.rst new file mode 100644 index 00000000000..0098124344d --- /dev/null +++ b/docs/user/ppl/cmd/replace.rst @@ -0,0 +1,270 @@ +============= +replace +============= + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + + +Description +============ +Using ``replace`` command to replace text in one or more fields. Supports literal string replacement and wildcard patterns using ``*``. + +Note: This command is only available when Calcite engine is enabled. + + +Syntax +============ +replace '' WITH '' [, '' WITH '']... IN [, ]... + + +Examples +======== + +Example 1: Replace text in one field +------------------------------------ + +The example shows replacing text in one field. + +PPL query:: + + os> source=accounts | replace "IL" WITH "Illinois" IN state | fields state; + fetched rows / total rows = 4/4 + +----------+ + | state | + |----------| + | Illinois | + | TN | + | VA | + | MD | + +----------+ + + +Example 2: Replace text in multiple fields +------------------------------------ + +The example shows replacing text in multiple fields. + +PPL query:: + + os> source=accounts | replace "IL" WITH "Illinois" IN state, address | fields state, address; + fetched rows / total rows = 4/4 + +----------+----------------------+ + | state | address | + |----------+----------------------| + | Illinois | 880 Holmes Lane | + | TN | 671 Bristol Street | + | VA | 789 Madison Street | + | MD | 467 Hutchinson Court | + +----------+----------------------+ + + +Example 3: Replace with other commands in a pipeline +------------------------------------ + +The example shows using replace with other commands in a query pipeline. + +PPL query:: + + os> source=accounts | replace "IL" WITH "Illinois" IN state | where age > 30 | fields state, age; + fetched rows / total rows = 3/3 + +----------+-----+ + | state | age | + |----------+-----| + | Illinois | 32 | + | TN | 36 | + | MD | 33 | + +----------+-----+ + +Example 4: Replace with multiple pattern/replacement pairs +------------------------------------ + +The example shows using multiple pattern/replacement pairs in a single replace command. The replacements are applied sequentially. + +PPL query:: + + os> source=accounts | replace "IL" WITH "Illinois", "TN" WITH "Tennessee" IN state | fields state; + fetched rows / total rows = 4/4 + +-----------+ + | state | + |-----------| + | Illinois | + | Tennessee | + | VA | + | MD | + +-----------+ + +Example 5: Pattern matching with LIKE and replace +------------------------------------ + +Since replace command only supports plain string literals, you can use LIKE command with replace for pattern matching needs. + +PPL query:: + + os> source=accounts | where LIKE(address, '%Holmes%') | replace "Holmes" WITH "HOLMES" IN address | fields address, state, gender, age, city; + fetched rows / total rows = 1/1 + +-----------------+-------+--------+-----+--------+ + | address | state | gender | age | city | + |-----------------+-------+--------+-----+--------| + | 880 HOLMES Lane | IL | M | 32 | Brogan | + +-----------------+-------+--------+-----+--------+ + + +Example 6: Wildcard suffix match +--------------------------------- + +Replace values that end with a specific pattern. The wildcard ``*`` matches any prefix. + +PPL query:: + + os> source=accounts | replace "*IL" WITH "Illinois" IN state | fields state; + fetched rows / total rows = 4/4 + +----------+ + | state | + |----------| + | Illinois | + | TN | + | VA | + | MD | + +----------+ + + +Example 7: Wildcard prefix match +--------------------------------- + +Replace values that start with a specific pattern. The wildcard ``*`` matches any suffix. + +PPL query:: + + os> source=accounts | replace "IL*" WITH "Illinois" IN state | fields state; + fetched rows / total rows = 4/4 + +----------+ + | state | + |----------| + | Illinois | + | TN | + | VA | + | MD | + +----------+ + + +Example 8: Wildcard capture and substitution +--------------------------------------------- + +Use wildcards in both pattern and replacement to capture and reuse matched portions. The number of wildcards must match in pattern and replacement. + +PPL query:: + + os> source=accounts | replace "* Lane" WITH "Lane *" IN address | fields address; + fetched rows / total rows = 4/4 + +----------------------+ + | address | + |----------------------| + | Lane 880 Holmes | + | 671 Bristol Street | + | 789 Madison Street | + | 467 Hutchinson Court | + +----------------------+ + + +Example 9: Multiple wildcards for pattern transformation +--------------------------------------------------------- + +Use multiple wildcards to transform patterns. Each wildcard in the replacement substitutes the corresponding captured value. + +PPL query:: + + os> source=accounts | replace "* *" WITH "*_*" IN address | fields address; + fetched rows / total rows = 4/4 + +----------------------+ + | address | + |----------------------| + | 880_Holmes Lane | + | 671_Bristol Street | + | 789_Madison Street | + | 467_Hutchinson Court | + +----------------------+ + + +Example 10: Wildcard with zero wildcards in replacement +-------------------------------------------------------- + +When replacement has zero wildcards, all matching values are replaced with the literal replacement string. + +PPL query:: + + os> source=accounts | replace "*IL*" WITH "Illinois" IN state | fields state; + fetched rows / total rows = 4/4 + +----------+ + | state | + |----------| + | Illinois | + | TN | + | VA | + | MD | + +----------+ + + +Example 11: Matching literal asterisks +--------------------------------------- + +Use ``\*`` to match literal asterisk characters (``\*`` = literal asterisk, ``\\`` = literal backslash). + +PPL query:: + + os> source=accounts | eval note = 'price: *sale*' | replace 'price: \*sale\*' WITH 'DISCOUNTED' IN note | fields note; + fetched rows / total rows = 4/4 + +------------+ + | note | + |------------| + | DISCOUNTED | + | DISCOUNTED | + | DISCOUNTED | + | DISCOUNTED | + +------------+ + +Example 12: Wildcard with no replacement wildcards +---------------------------------------------------- + +Use wildcards in pattern but none in replacement to create a fixed output. + +PPL query:: + + os> source=accounts | eval test = 'prefix-value-suffix' | replace 'prefix-*-suffix' WITH 'MATCHED' IN test | fields test; + fetched rows / total rows = 4/4 + +---------+ + | test | + |---------| + | MATCHED | + | MATCHED | + | MATCHED | + | MATCHED | + +---------+ + +Example 13: Escaped asterisks with wildcards +--------------------------------------------- + +Combine escaped asterisks (literal) with wildcards for complex patterns. + +PPL query:: + + os> source=accounts | eval label = 'file123.txt' | replace 'file*.*' WITH '\**.*' IN label | fields label; + fetched rows / total rows = 4/4 + +----------+ + | label | + |----------| + | *123.txt | + | *123.txt | + | *123.txt | + | *123.txt | + +----------+ + + +Limitations +=========== +* Wildcards: ``*`` matches zero or more characters (case-sensitive) +* Replacement wildcards must match pattern wildcard count, or be zero +* Escape sequences: ``\*`` (literal asterisk), ``\\`` (literal backslash) \ No newline at end of file diff --git a/docs/user/ppl/cmd/rex.rst b/docs/user/ppl/cmd/rex.rst index 8841f0cb1d7..28839247194 100644 --- a/docs/user/ppl/cmd/rex.rst +++ b/docs/user/ppl/cmd/rex.rst @@ -166,7 +166,7 @@ Demonstrates naming restrictions for capture groups. Group names cannot contain Invalid PPL query with underscores:: os> source=accounts | rex field=email "(?[^@]+)@(?[^.]+)" | fields email, user_name, email_domain ; - {'reason': 'Invalid Query', 'details': 'Rex pattern must contain at least one named capture group', 'type': 'IllegalArgumentException'} + {'reason': 'Invalid Query', 'details': "Invalid capture group name 'user_name'. Java regex group names must start with a letter and contain only letters and digits.", 'type': 'IllegalArgumentException'} Error: Query returned no data Correct PPL query without underscores:: @@ -206,17 +206,17 @@ PPL query exceeding the configured limit results in an error:: Comparison with Related Commands ================================ -============================= ============ ============ -Feature rex parse -============================= ============ ============ -Pattern Type Java Regex Java Regex -Named Groups Required Yes Yes -Multiple Named Groups Yes No -Multiple Matches Yes No -Text Substitution Yes No -Offset Tracking Yes No -Underscores in Group Names No No -============================= ============ ============ +================================== ============ ============ +Feature rex parse +================================== ============ ============ +Pattern Type Java Regex Java Regex +Named Groups Required Yes Yes +Multiple Named Groups Yes No +Multiple Matches Yes No +Text Substitution Yes No +Offset Tracking Yes No +Special Characters in Group Names No No +================================== ============ ============ Limitations @@ -226,7 +226,6 @@ There are several important limitations with the rex command: **Named Capture Group Naming:** -- Named capture groups cannot contain underscores due to Java regex limitations - Group names must start with a letter and contain only letters and digits - For detailed Java regex pattern syntax and usage, refer to the `official Java Pattern documentation `_ diff --git a/docs/user/ppl/cmd/search.rst b/docs/user/ppl/cmd/search.rst index 44cd2377ce0..11b6bf99df4 100644 --- a/docs/user/ppl/cmd/search.rst +++ b/docs/user/ppl/cmd/search.rst @@ -124,7 +124,7 @@ Field Types and Search Behavior * ``search client_ip="192.168.1.0/24" source=logs`` -* Limitations: No wildcards for partial IP matching +* Limitations: No wildcards for partial IP matching. For wildcard search use multi field with keyword: ``search ip_address.keyword='1*' source=logs`` or WHERE clause: ``source=logs | where cast(ip_address as string) like '1%'`` **Field Type Performance Tips**: diff --git a/docs/user/ppl/cmd/sort.rst b/docs/user/ppl/cmd/sort.rst index c9750bab079..e02a8fdae8d 100644 --- a/docs/user/ppl/cmd/sort.rst +++ b/docs/user/ppl/cmd/sort.rst @@ -16,13 +16,16 @@ Description Syntax ============ -sort [count] <[+|-] sort-field>... [asc|a|desc|d] +sort [count] <[+|-] sort-field | sort-field [asc|a|desc|d]>... * count (Since 3.3): optional. The number of results to return. **Default:** returns all results. Specifying a count of 0 or less than 0 also returns all results. * [+|-]: optional. The plus [+] stands for ascending order and NULL/MISSING first and a minus [-] stands for descending order and NULL/MISSING last. **Default:** ascending order and NULL/MISSING first. +* [asc|a|desc|d]: optional. asc/a stands for ascending order and NULL/MISSING first. desc/d stands for descending order and NULL/MISSING last. **Default:** ascending order and NULL/MISSING first. * sort-field: mandatory. The field used to sort. Can use ``auto(field)``, ``str(field)``, ``ip(field)``, or ``num(field)`` to specify how to interpret field values. -* [asc|a|desc|d] (Since 3.3): optional. asc/a keeps the sort order as specified. desc/d reverses the sort results. If multiple fields are specified with desc/d, reverses order of the first field then for all duplicate values of the first field, reverses the order of the values of the second field and so on. **Default:** asc. + +.. note:: + You cannot mix +/- and asc/desc in the same sort command. Choose one approach for all fields in a single sort command. Example 1: Sort by one field @@ -63,10 +66,10 @@ PPL query:: +----------------+-----+ -Example 3: Sort by one field in descending order -================================================ +Example 3: Sort by one field in descending order (using -) +========================================================== -The example show sort all the document with age field in descending order. +The example show sort all the document with age field in descending order using the - operator. PPL query:: @@ -81,10 +84,28 @@ PPL query:: | 13 | 28 | +----------------+-----+ -Example 4: Sort by multiple field -============================= +Example 4: Sort by one field in descending order (using desc) +============================================================== -The example show sort all the document with gender field in ascending order and age field in descending. +The example show sort all the document with age field in descending order using the desc keyword. + +PPL query:: + + os> source=accounts | sort age desc | fields account_number, age; + fetched rows / total rows = 4/4 + +----------------+-----+ + | account_number | age | + |----------------+-----| + | 6 | 36 | + | 18 | 33 | + | 1 | 32 | + | 13 | 28 | + +----------------+-----+ + +Example 5: Sort by multiple fields (using +/-) +============================================== + +The example show sort all the document with gender field in ascending order and age field in descending using +/- operators. PPL query:: @@ -99,10 +120,28 @@ PPL query:: | 1 | M | 32 | +----------------+--------+-----+ -Example 4: Sort by field include null value +Example 6: Sort by multiple fields (using asc/desc) +==================================================== + +The example show sort all the document with gender field in ascending order and age field in descending using asc/desc keywords. + +PPL query:: + + os> source=accounts | sort gender asc, age desc | fields account_number, gender, age; + fetched rows / total rows = 4/4 + +----------------+--------+-----+ + | account_number | gender | age | + |----------------+--------+-----| + | 13 | F | 28 | + | 6 | M | 36 | + | 18 | M | 33 | + | 1 | M | 32 | + +----------------+--------+-----+ + +Example 7: Sort by field include null value =========================================== -The example show sort employer field by default option (ascending order and null first), the result show that null value is in the first row. +The example shows sorting the employer field by the default option (ascending order and null first), the result shows that the null value is in the first row. PPL query:: @@ -117,7 +156,7 @@ PPL query:: | Quility | +----------+ -Example 5: Specify the number of sorted documents to return +Example 8: Specify the number of sorted documents to return ============================================================ The example shows sorting all the document and returning 2 documents. @@ -133,7 +172,7 @@ PPL query:: | 1 | 32 | +----------------+-----+ -Example 6: Sort with desc modifier +Example 9: Sort with desc modifier =================================== The example shows sorting with the desc modifier to reverse sort order. @@ -151,26 +190,7 @@ PPL query:: | 13 | 28 | +----------------+-----+ -Example 7: Sort by multiple fields with desc modifier -====================================================== - -The example shows sorting by multiple fields using desc, which reverses the sort order for all specified fields. Gender is reversed from ascending to descending, and the descending age sort is reversed to ascending within each gender group. - -PPL query:: - - os> source=accounts | sort gender, -age desc | fields account_number, gender, age; - fetched rows / total rows = 4/4 - +----------------+--------+-----+ - | account_number | gender | age | - |----------------+--------+-----| - | 1 | M | 32 | - | 18 | M | 33 | - | 6 | M | 36 | - | 13 | F | 28 | - +----------------+--------+-----+ - - -Example 8: Sort with specifying field type +Example 10: Sort with specifying field type ================================== The example shows sorting with str() to sort numeric values lexicographically. diff --git a/docs/user/ppl/cmd/stats.rst b/docs/user/ppl/cmd/stats.rst index e61b4120410..24b80d4675b 100644 --- a/docs/user/ppl/cmd/stats.rst +++ b/docs/user/ppl/cmd/stats.rst @@ -58,8 +58,8 @@ stats [bucket_nullable=bool] ... [by-clause] * span-expression: optional, at most one. - * Syntax: span(field_expr, interval_expr) - * Description: The unit of the interval expression is the natural unit by default. **If the field is a date/time type field, the aggregation results always ignore null bucket**. And the interval is in date/time units, you will need to specify the unit in the interval expression. For example, to split the field ``age`` into buckets by 10 years, it looks like ``span(age, 10)``. And here is another example of time span, the span to split a ``timestamp`` field into hourly intervals, it looks like ``span(timestamp, 1h)``. + * Syntax: span([field_expr,] interval_expr) + * Description: The unit of the interval expression is the natural unit by default. If ``field_expr`` is omitted, span will use the implicit ``@timestamp`` field. An error will be thrown if this field doesn't exist. **If the field is a date/time type field, the aggregation results always ignore null bucket**. And the interval is in date/time units, you will need to specify the unit in the interval expression. For example, to split the field ``age`` into buckets by 10 years, it looks like ``span(age, 10)``. And here is another example of time span, the span to split a ``timestamp`` field into hourly intervals, it looks like ``span(timestamp, 1h)``. * Available time unit: +----------------------------+ @@ -580,7 +580,7 @@ Description Version: 3.3.0 (Calcite engine only) -Usage: LIST(expr). Collects all values from the specified expression into an array. Values are converted to strings, nulls are filtered, and duplicates are preserved. +Usage: LIST(expr). Collects all values from the specified expression into an array. Values are converted to strings, nulls are filtered, and duplicates are preserved. The function returns up to 100 values with no guaranteed ordering. * expr: The field expression to collect values from. @@ -977,3 +977,18 @@ PPL query:: | 1 | 2025-01-01 | 2 | +-----+------------+--------+ + +Example 18: Calculate the count by the implicit @timestamp field +================================================================ + +This example demonstrates that if you omit the field parameter in the span function, it will automatically use the implicit ``@timestamp`` field. + +PPL query:: + + PPL> source=big5 | stats count() by span(1month) + fetched rows / total rows = 1/1 + +---------+---------------------+ + | count() | span(1month) | + |---------+---------------------| + | 1 | 2023-01-01 00:00:00 | + +---------+---------------------+ diff --git a/docs/user/ppl/cmd/streamstats.rst b/docs/user/ppl/cmd/streamstats.rst new file mode 100644 index 00000000000..0ac18637fec --- /dev/null +++ b/docs/user/ppl/cmd/streamstats.rst @@ -0,0 +1,229 @@ +=========== +streamstats +=========== + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + + +Description +=========== +The ``streamstats`` command is used to calculate cumulative or rolling statistics as events are processed in order. Unlike ``stats`` or ``eventstats`` which operate on the entire dataset at once, it computes values incrementally on a per-event basis, often respecting the order of events in the search results. It allows you to generate running totals, moving averages, and other statistics that evolve with the stream of events. + +Key aspects of `streamstats`: + +1. It computes statistics incrementally as each event is processed, making it suitable for time-series and sequence-based analysis. +2. Supports arguments such as window (for sliding window calculations) and current (to control whether the current event included in calculation). +3. Retains all original events and appends new fields containing the calculated statistics. +4. Particularly useful for calculating running totals, identifying trends, or detecting changes over sequences of events. + +Difference between ``stats``, ``eventstats`` and ``streamstats`` + +All of these commands can be used to generate aggregations such as average, sum, and maximum, but they have some key differences in how they operate and what they produce: + +* Transformation Behavior: + * ``stats``: Transforms all events into an aggregated result table, losing original event structure. + * ``eventstats``: Adds aggregation results as new fields to the original events without removing the event structure. + * ``streamstats``: Adds cumulative (running) aggregation results to each event as they stream through the pipeline. +* Output Format: + * ``stats``: Output contains only aggregated values. Original raw events are not preserved. + * ``eventstats``: Original events remain, with extra fields containing summary statistics. + * ``streamstats``: Original events remain, with extra fields containing running totals or cumulative statistics. +* Aggregation Scope: + * ``stats``: Based on all events in the search (or groups defined by BY clause). + * ``eventstats``: Based on all relevant events, then the result is added back to each event in the group. + * ``streamstats``: Calculations occur progressively as each event is processed; can be scoped by window. +* Use Cases: + * ``stats``: When only aggregated results are needed (e.g., counts, averages, sums). + * ``eventstats``: When aggregated statistics are needed alongside original event data. + * ``streamstats``: When a running total or cumulative statistic is needed across event streams. + +Syntax +====== +streamstats [current=] [window=] [global=] [reset_before="("")"] [reset_after="("")"] ... [by-clause] + +* function: mandatory. A aggregation function or window function. +* current: optional. If true, the search includes the given, or current, event in the summary calculations. If false, the search uses the field value from the previous event. Syntax: current=. **Default:** true. +* window: optional. Specifies the number of events to use when computing the statistics. Syntax: window=. **Default:** 0, which means that all previous and current events are used. +* global: optional. Used only when the window argument is set. Defines whether to use a single window, global=true, or to use separate windows based on the by clause. If global=false and window is set to a non-zero value, a separate window is used for each group of values of the field specified in the by clause. Syntax: global=. **Default:** true. +* reset_before: optional. Before streamstats calculates for an event, reset_before resets all accumulated statistics when the eval-expression evaluates to true. If used with window, the window is also reset. Syntax: reset_before="("")". **Default:** false. +* reset_after: optional. After streamstats calculations for an event, reset_after resets all accumulated statistics when the eval-expression evaluates to true. This expression can reference fields returned by streamstats. If used with window, the window is also reset. Syntax: reset_after="("")". **Default:** false. +* by-clause: optional. The by clause could be the fields and expressions like scalar functions and aggregation functions. Besides, the span clause can be used to split specific field into buckets in the same interval, the stats then does the aggregation by these span buckets. Syntax: by [span-expression,] [field,]... **Default:** If no is specified, all events are processed as a single group and running statistics are computed across the entire event stream. +* span-expression: optional, at most one. Splits field into buckets by intervals. Syntax: span(field_expr, interval_expr). For example, ``span(age, 10)`` creates 10-year age buckets, ``span(timestamp, 1h)`` creates hourly buckets. + * Available time units: + * millisecond (ms) + * second (s) + * minute (m, case sensitive) + * hour (h) + * day (d) + * week (w) + * month (M, case sensitive) + * quarter (q) + * year (y) + +Aggregation Functions +===================== + +The streamstats command supports the following aggregation functions: + +* COUNT: Count of values +* SUM: Sum of numeric values +* AVG: Average of numeric values +* MAX: Maximum value +* MIN: Minimum value +* VAR_SAMP: Sample variance +* VAR_POP: Population variance +* STDDEV_SAMP: Sample standard deviation +* STDDEV_POP: Population standard deviation +* DISTINCT_COUNT/DC: Distinct count of values +* EARLIEST: Earliest value by timestamp +* LATEST: Latest value by timestamp + +For detailed documentation of each function, see `Aggregation Functions <../functions/aggregation.rst>`_. + +Usage +===== + +Streamstats:: + + source = table | streamstats avg(a) + source = table | streamstats current = false avg(a) + source = table | streamstats window = 5 sum(b) + source = table | streamstats current = false window = 2 max(a) + source = table | where a < 50 | streamstats count(c) + source = table | streamstats min(c), max(c) by b + source = table | streamstats count(c) as count_by by b | where count_by > 1000 + source = table | streamstats dc(field) as distinct_count + source = table | streamstats distinct_count(category) by region + source = table | streamstats current=false window=2 global=false avg(a) by b + source = table | streamstats window=2 reset_before=a>31 avg(b) + source = table | streamstats current=false reset_after=a>31 avg(b) by c + + +Example 1: Calculate the running average, sum, and count of a field by group +============================================================================ + +This example calculates the running average age, running sum of age, and running count of events for all the accounts, grouped by gender. + +PPL query:: + + os> source=accounts | streamstats avg(age) as running_avg, sum(age) as running_sum, count() as running_count by gender; + fetched rows / total rows = 4/4 + +----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------+--------------------+-------------+---------------+ + | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | running_avg | running_sum | running_count | + |----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------+--------------------+-------------+---------------| + | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | 32.0 | 32 | 1 | + | 6 | Hattie | 671 Bristol Street | 5686 | M | Dante | Netagy | TN | 36 | hattiebond@netagy.com | Bond | 34.0 | 68 | 2 | + | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | 28.0 | 28 | 1 | + | 18 | Dale | 467 Hutchinson Court | 4180 | M | Orick | null | MD | 33 | daleadams@boink.com | Adams | 33.666666666666664 | 101 | 3 | + +----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------+--------------------+-------------+---------------+ + + +Example 2: Running maximum age over a 2-row window +================================================== + +This example calculates the running maximum age over a 2-row window, excluding the current event. + +PPL query:: + + os> source=state_country | streamstats current=false window=2 max(age) as prev_max_age + fetched rows / total rows = 8/8 + +-------+---------+------------+-------+------+-----+--------------+ + | name | country | state | month | year | age | prev_max_age | + |-------+---------+------------+-------+------+-----+--------------| + | Jake | USA | California | 4 | 2023 | 70 | null | + | Hello | USA | New York | 4 | 2023 | 30 | 70 | + | John | Canada | Ontario | 4 | 2023 | 25 | 70 | + | Jane | Canada | Quebec | 4 | 2023 | 20 | 30 | + | Jim | Canada | B.C | 4 | 2023 | 27 | 25 | + | Peter | Canada | B.C | 4 | 2023 | 57 | 27 | + | Rick | Canada | B.C | 4 | 2023 | 70 | 57 | + | David | USA | Washington | 4 | 2023 | 40 | 70 | + +-------+---------+------------+-------+------+-----+--------------+ + + +Example 3: Use the global argument to calculate running statistics +================================================================== + +The global argument is only applicable when a window argument is set. It defines how the window is applied in relation to the grouping fields: + +* global=true: a global window is applied across all rows, but the calculations inside the window still respect the by groups. +* global=false: the window itself is created per group, meaning each group gets its own independent window. + +This example shows how to calculate the running average of age across accounts by country, using global argument. + +original data:: + + +-------+---------+------------+-------+------+-----+ + | name | country | state | month | year | age | + |-------+---------+------------+-------+------+-----+ + | Jake | USA | California | 4 | 2023 | 70 | + | Hello | USA | New York | 4 | 2023 | 30 | + | John | Canada | Ontario | 4 | 2023 | 25 | + | Jane | Canada | Quebec | 4 | 2023 | 20 | + | Jim | Canada | B.C | 4 | 2023 | 27 | + | Peter | Canada | B.C | 4 | 2023 | 57 | + | Rick | Canada | B.C | 4 | 2023 | 70 | + | David | USA | Washington | 4 | 2023 | 40 | + +-------+---------+------------+-------+------+-----+ + +* global=true: The window slides across all rows globally (following their input order), but inside each window, aggregation is still computed by country. So we process the data stream row by row to build the sliding window with size 2. We can see that David and Rick are in a window. +* global=false: Each by group (country) forms its own independent stream and window (size 2). So David and Hello are in one window for USA. This time we get running_avg 35 for David, rather than 40 when global is set true. + +PPL query:: + + os> source=state_country | streamstats window=2 global=true avg(age) as running_avg by country ; + fetched rows / total rows = 8/8 + +-------+---------+------------+-------+------+-----+-------------+ + | name | country | state | month | year | age | running_avg | + |-------+---------+------------+-------+------+-----+-------------| + | Jake | USA | California | 4 | 2023 | 70 | 70.0 | + | Hello | USA | New York | 4 | 2023 | 30 | 50.0 | + | John | Canada | Ontario | 4 | 2023 | 25 | 25.0 | + | Jane | Canada | Quebec | 4 | 2023 | 20 | 22.5 | + | Jim | Canada | B.C | 4 | 2023 | 27 | 23.5 | + | Peter | Canada | B.C | 4 | 2023 | 57 | 42.0 | + | Rick | Canada | B.C | 4 | 2023 | 70 | 63.5 | + | David | USA | Washington | 4 | 2023 | 40 | 40.0 | + +-------+---------+------------+-------+------+-----+-------------+ + + os> source=state_country | streamstats window=2 global=false avg(age) as running_avg by country ; + fetched rows / total rows = 8/8 + +-------+---------+------------+-------+------+-----+-------------+ + | name | country | state | month | year | age | running_avg | + |-------+---------+------------+-------+------+-----+-------------| + | Jake | USA | California | 4 | 2023 | 70 | 70.0 | + | Hello | USA | New York | 4 | 2023 | 30 | 50.0 | + | John | Canada | Ontario | 4 | 2023 | 25 | 25.0 | + | Jane | Canada | Quebec | 4 | 2023 | 20 | 22.5 | + | Jim | Canada | B.C | 4 | 2023 | 27 | 23.5 | + | Peter | Canada | B.C | 4 | 2023 | 57 | 42.0 | + | Rick | Canada | B.C | 4 | 2023 | 70 | 63.5 | + | David | USA | Washington | 4 | 2023 | 40 | 35.0 | + +-------+---------+------------+-------+------+-----+-------------+ + + +Example 4: Use the reset_before and reset_after arguments to reset statistics +============================================================================= + +This example calculates the running average of age across accounts by country, with resets applied. + +PPL query:: + + os> source=state_country | streamstats current=false reset_before=age>34 reset_after=age<25 avg(age) as avg_age by country; + fetched rows / total rows = 8/8 + +-------+---------+------------+-------+------+-----+---------+ + | name | country | state | month | year | age | avg_age | + |-------+---------+------------+-------+------+-----+---------| + | Jake | USA | California | 4 | 2023 | 70 | null | + | Hello | USA | New York | 4 | 2023 | 30 | 70.0 | + | John | Canada | Ontario | 4 | 2023 | 25 | null | + | Jane | Canada | Quebec | 4 | 2023 | 20 | 25.0 | + | Jim | Canada | B.C | 4 | 2023 | 27 | null | + | Peter | Canada | B.C | 4 | 2023 | 57 | null | + | Rick | Canada | B.C | 4 | 2023 | 70 | null | + | David | USA | Washington | 4 | 2023 | 40 | null | + +-------+---------+------------+-------+------+-----+---------+ \ No newline at end of file diff --git a/docs/user/ppl/cmd/subquery.rst b/docs/user/ppl/cmd/subquery.rst index 534db112b7e..f7883202c70 100644 --- a/docs/user/ppl/cmd/subquery.rst +++ b/docs/user/ppl/cmd/subquery.rst @@ -42,6 +42,10 @@ RelationSubquery:: Configuration ============= + +plugins.calcite.enabled +----------------------- + This command requires Calcite enabled. In 3.0.0-beta, as an experimental the Calcite configuration is disabled by default. Enable Calcite:: @@ -66,6 +70,31 @@ Result set:: "transient": {} } +plugins.ppl.subsearch.maxout +---------------------------- + +The size configures the maximum of rows to return from subsearch. The default value is: ``10000``. A value of ``0`` indicates that the restriction is unlimited. + +Change the subsearch.maxout to unlimited:: + + sh$ curl -sS -H 'Content-Type: application/json' \ + ... -X PUT localhost:9200/_plugins/_query/settings \ + ... -d '{"persistent" : {"plugins.ppl.subsearch.maxout" : "0"}}' + { + "acknowledged": true, + "persistent": { + "plugins": { + "ppl": { + "subsearch": { + "maxout": "-1" + } + } + } + }, + "transient": {} + } + + Usage ===== diff --git a/docs/user/ppl/cmd/timechart.rst b/docs/user/ppl/cmd/timechart.rst index a5708769035..512fa76370c 100644 --- a/docs/user/ppl/cmd/timechart.rst +++ b/docs/user/ppl/cmd/timechart.rst @@ -57,14 +57,50 @@ Syntax * When set to true, values beyond the limit are grouped into an "OTHER" category. * Only applies when using the "by" clause and when there are more distinct values than the limit. +* **by**: optional. Groups the results by the specified field in addition to time intervals. + + * If not specified, the aggregation is performed across all documents in each time interval. + * **aggregation_function**: mandatory. The aggregation function to apply to each time bucket. * Currently, only a single aggregation function is supported. - * Available functions: All aggregation functions supported by the :doc:`stats ` command are supported. + * Available functions: All aggregation functions supported by the :doc:`stats ` command, as well as the timechart-specific aggregations listed below. -* **by**: optional. Groups the results by the specified field in addition to time intervals. +PER_SECOND +---------- - * If not specified, the aggregation is performed across all documents in each time interval. +Usage: per_second(field) calculates the per-second rate for a numeric field within each time bucket. + +The calculation formula is: `per_second(field) = sum(field) / span_in_seconds`, where `span_in_seconds` is the span interval in seconds. + +Return type: DOUBLE + +PER_MINUTE +---------- + +Usage: per_minute(field) calculates the per-minute rate for a numeric field within each time bucket. + +The calculation formula is: `per_minute(field) = sum(field) * 60 / span_in_seconds`, where `span_in_seconds` is the span interval in seconds. + +Return type: DOUBLE + +PER_HOUR +-------- + +Usage: per_hour(field) calculates the per-hour rate for a numeric field within each time bucket. + +The calculation formula is: `per_hour(field) = sum(field) * 3600 / span_in_seconds`, where `span_in_seconds` is the span interval in seconds. + +Return type: DOUBLE + +PER_DAY +------- + +Usage: per_day(field) calculates the per-day rate for a numeric field within each time bucket. + +The calculation formula is: `per_day(field) = sum(field) * 86400 / span_in_seconds`, where `span_in_seconds` is the span interval in seconds. + +Return type: DOUBLE Notes ===== @@ -332,3 +368,20 @@ PPL query:: | 2024-07-01 00:00:00 | null | 1 | +---------------------+--------+-------+ +Example 11: Calculate packets per second rate +============================================= + +This example calculates the per-second packet rate for network traffic data using the per_second() function. + +PPL query:: + + os> source=events | timechart span=30m per_second(packets) by host + fetched rows / total rows = 4/4 + +---------------------+---------+---------------------+ + | @timestamp | host | per_second(packets) | + |---------------------+---------+---------------------| + | 2023-01-01 10:00:00 | server1 | 0.1 | + | 2023-01-01 10:00:00 | server2 | 0.05 | + | 2023-01-01 10:30:00 | server1 | 0.1 | + | 2023-01-01 10:30:00 | server2 | 0.05 | + +---------------------+---------+---------------------+ diff --git a/docs/user/ppl/cmd/top.rst b/docs/user/ppl/cmd/top.rst index 5f4bfb9b4b6..a786d7ed9a9 100644 --- a/docs/user/ppl/cmd/top.rst +++ b/docs/user/ppl/cmd/top.rst @@ -1,6 +1,6 @@ -============= +=== top -============= +=== .. rubric:: Table of contents @@ -10,12 +10,12 @@ top Description -============ +=========== | Using ``top`` command to find the most common tuple of values of all fields in the field list. Syntax -============ +====== top [N] [by-clause] top [N] [top-options] [by-clause] ``(available from 3.1.0+)`` @@ -26,10 +26,13 @@ top [N] [top-options] [by-clause] ``(available from 3.1.0+)`` * top-options: optional. options for the top command. Supported syntax is [countfield=] [showcount=]. * showcount=: optional. whether to create a field in output that represent a count of the tuple of values. Default value is ``true``. * countfield=: optional. the name of the field that contains count. Default value is ``'count'``. +* usenull=: optional (since 3.4.0). whether to output the null value. The default value of ``usenull`` is determined by ``plugins.ppl.syntax.legacy.preferred``: + * When ``plugins.ppl.syntax.legacy.preferred=true``, ``usenull`` defaults to ``true`` + * When ``plugins.ppl.syntax.legacy.preferred=false``, ``usenull`` defaults to ``false`` Example 1: Find the most common values in a field -=========================================== +================================================= The example finds most common gender of all the accounts. @@ -45,7 +48,7 @@ PPL query:: +--------+ Example 2: Find the most common values in a field -=========================================== +================================================= The example finds most common gender of all the accounts. @@ -60,7 +63,7 @@ PPL query:: +--------+ Example 2: Find the most common values organized by gender -==================================================== +========================================================== The example finds most common age of all the accounts group by gender. @@ -78,12 +81,10 @@ PPL query:: Example 3: Top command with Calcite enabled =========================================== -The example finds most common gender of all the accounts when ``plugins.calcite.enabled`` is true. - PPL query:: - PPL> source=accounts | top gender; - fetched row + os> source=accounts | top gender; + fetched rows / total rows = 2/2 +--------+-------+ | gender | count | |--------+-------| @@ -95,12 +96,10 @@ PPL query:: Example 4: Specify the count field option ========================================= -The example specifies the count field when ``plugins.calcite.enabled`` is true. - PPL query:: - PPL> source=accounts | top countfield='cnt' gender; - fetched row + os> source=accounts | top countfield='cnt' gender; + fetched rows / total rows = 2/2 +--------+-----+ | gender | cnt | |--------+-----| @@ -108,6 +107,36 @@ PPL query:: | F | 1 | +--------+-----+ + +Example 5: Specify the usenull field option +=========================================== + +PPL query:: + + os> source=accounts | top usenull=false email; + fetched rows / total rows = 3/3 + +-----------------------+-------+ + | email | count | + |-----------------------+-------| + | amberduke@pyrami.com | 1 | + | daleadams@boink.com | 1 | + | hattiebond@netagy.com | 1 | + +-----------------------+-------+ + +PPL query:: + + os> source=accounts | top usenull=true email; + fetched rows / total rows = 4/4 + +-----------------------+-------+ + | email | count | + |-----------------------+-------| + | null | 1 | + | amberduke@pyrami.com | 1 | + | daleadams@boink.com | 1 | + | hattiebond@netagy.com | 1 | + +-----------------------+-------+ + + Limitations =========== The ``top`` command is not rewritten to OpenSearch DSL, it is only executed on the coordination node. diff --git a/docs/user/ppl/cmd/where.rst b/docs/user/ppl/cmd/where.rst index 115bffe7de5..9bdb8a75aa3 100644 --- a/docs/user/ppl/cmd/where.rst +++ b/docs/user/ppl/cmd/where.rst @@ -10,18 +10,21 @@ where Description -============ +=========== | The ``where`` command bool-expression to filter the search result. The ``where`` command only return the result when bool-expression evaluated to true. Syntax -============ +====== where * bool-expression: optional. any expression which could be evaluated to boolean value. +Examples +======== + Example 1: Filter result set with condition -=========================================== +-------------------------------------------- The example show fetch all the document from accounts index with . @@ -36,3 +39,131 @@ PPL query:: | 13 | F | +----------------+--------+ +Example 2: Basic Field Comparison +---------------------------------- + +The example shows how to filter accounts with balance greater than 30000. + +PPL query:: + + os> source=accounts | where balance > 30000 | fields account_number, balance; + fetched rows / total rows = 2/2 + +----------------+---------+ + | account_number | balance | + |----------------+---------| + | 1 | 39225 | + | 13 | 32838 | + +----------------+---------+ + +Example 3: Pattern Matching with LIKE +-------------------------------------- + +Pattern Matching with Underscore (_) + +The example demonstrates using LIKE with underscore (_) to match a single character. + +PPL query:: + + os> source=accounts | where LIKE(state, 'M_') | fields account_number, state; + fetched rows / total rows = 1/1 + +----------------+-------+ + | account_number | state | + |----------------+-------| + | 18 | MD | + +----------------+-------+ + +Pattern Matching with Percent (%) + +The example demonstrates using LIKE with percent (%) to match multiple characters. + +PPL query:: + + os> source=accounts | where LIKE(state, 'V%') | fields account_number, state; + fetched rows / total rows = 1/1 + +----------------+-------+ + | account_number | state | + |----------------+-------| + | 13 | VA | + +----------------+-------+ + +Example 4: Multiple Conditions +------------------------------- + +The example shows how to combine multiple conditions using AND operator. + +PPL query:: + + os> source=accounts | where age > 30 AND gender = 'M' | fields account_number, age, gender; + fetched rows / total rows = 3/3 + +----------------+-----+--------+ + | account_number | age | gender | + |----------------+-----+--------| + | 1 | 32 | M | + | 6 | 36 | M | + | 18 | 33 | M | + +----------------+-----+--------+ + +Example 5: Using IN Operator +----------------------------- + +The example demonstrates using IN operator to match multiple values. + +PPL query:: + + os> source=accounts | where state IN ('IL', 'VA') | fields account_number, state; + fetched rows / total rows = 2/2 + +----------------+-------+ + | account_number | state | + |----------------+-------| + | 1 | IL | + | 13 | VA | + +----------------+-------+ + +Example 6: NULL Checks +---------------------- + +The example shows how to filter records with NULL values. + +PPL query:: + + os> source=accounts | where ISNULL(employer) | fields account_number, employer; + fetched rows / total rows = 1/1 + +----------------+----------+ + | account_number | employer | + |----------------+----------| + | 18 | null | + +----------------+----------+ + +Example 7: Complex Conditions +------------------------------ + +The example demonstrates combining multiple conditions with parentheses and logical operators. + +PPL query:: + + os> source=accounts | where (balance > 40000 OR age > 35) AND gender = 'M' | fields account_number, balance, age, gender; + fetched rows / total rows = 1/1 + +----------------+---------+-----+--------+ + | account_number | balance | age | gender | + |----------------+---------+-----+--------| + | 6 | 5686 | 36 | M | + +----------------+---------+-----+--------+ + +Example 8: NOT Conditions +-------------------------- + +The example shows how to use NOT operator to exclude matching records. + +PPL query:: + + os> source=accounts | where NOT state = 'CA' | fields account_number, state; + fetched rows / total rows = 4/4 + +----------------+-------+ + | account_number | state | + |----------------+-------| + | 1 | IL | + | 6 | TN | + | 13 | VA | + | 18 | MD | + +----------------+-------+ + diff --git a/docs/user/ppl/functions/collection.rst b/docs/user/ppl/functions/collection.rst index 95d55fa7e2d..76931e53876 100644 --- a/docs/user/ppl/functions/collection.rst +++ b/docs/user/ppl/functions/collection.rst @@ -14,8 +14,6 @@ ARRAY Description >>>>>>>>>>> -Version: 3.1.0 - Usage: ``array(value1, value2, value3...)`` create an array with input values. Currently we don't allow mixture types. We will infer a least restricted type, for example ``array(1, "demo")`` -> ["1", "demo"] Argument type: value1: ANY, value2: ANY, ... @@ -24,21 +22,21 @@ Return type: ARRAY Example:: - PPL> source=people | eval array = array(1, 2, 3) | fields array | head 1 + os> source=people | eval array = array(1, 2, 3) | fields array | head 1 fetched rows / total rows = 1/1 - +----------------------------------+ - | array | - |----------------------------------| - | [1, 2, 3] | - +----------------------------------+ + +---------+ + | array | + |---------| + | [1,2,3] | + +---------+ - PPL> source=people | eval array = array(1, "demo") | fields array | head 1 + os> source=people | eval array = array(1, "demo") | fields array | head 1 fetched rows / total rows = 1/1 - +----------------------------------+ - | array | - |----------------------------------| - | ["1", "demo"] | - +----------------------------------+ + +----------+ + | array | + |----------| + | [1,demo] | + +----------+ ARRAY_LENGTH ------------ @@ -46,8 +44,6 @@ ARRAY_LENGTH Description >>>>>>>>>>> -Version: 3.1.0 - Usage: ``array_length(array)`` returns the length of input array. Argument type: array:ARRAY @@ -56,13 +52,13 @@ Return type: INTEGER Example:: - PPL> source=people | eval array = array(1, 2, 3) | eval length = array_length(array) | fields length | head 1 + os> source=people | eval array = array(1, 2, 3) | eval length = array_length(array) | fields length | head 1 fetched rows / total rows = 1/1 - +---------------+ - | length | - |---------------| - | 4 | - +---------------+ + +--------+ + | length | + |--------| + | 3 | + +--------+ FORALL ------ @@ -70,8 +66,6 @@ FORALL Description >>>>>>>>>>> -Version: 3.1.0 - Usage: ``forall(array, function)`` check whether all element inside array can meet the lambda function. The function should also return boolean. The lambda function accepts one single input. Argument type: array:ARRAY, function:LAMBDA @@ -80,13 +74,13 @@ Return type: BOOLEAN Example:: - PPL> source=people | eval array = array(1, 2, 3), result = forall(array, x -> x > 0) | fields result | head 1 + os> source=people | eval array = array(1, 2, 3), result = forall(array, x -> x > 0) | fields result | head 1 fetched rows / total rows = 1/1 - +---------+ - | result | - |---------| - | true | - +---------+ + +--------+ + | result | + |--------| + | True | + +--------+ EXISTS ------ @@ -94,8 +88,6 @@ EXISTS Description >>>>>>>>>>> -Version: 3.1.0 - Usage: ``exists(array, function)`` check whether existing one of element inside array can meet the lambda function. The function should also return boolean. The lambda function accepts one single input. Argument type: array:ARRAY, function:LAMBDA @@ -104,13 +96,13 @@ Return type: BOOLEAN Example:: - PPL> source=people | eval array = array(-1, -2, 3), result = exists(array, x -> x > 0) | fields result | head 1 + os> source=people | eval array = array(-1, -2, 3), result = exists(array, x -> x > 0) | fields result | head 1 fetched rows / total rows = 1/1 - +---------+ - | result | - |---------| - | true | - +---------+ + +--------+ + | result | + |--------| + | True | + +--------+ FILTER ------ @@ -118,8 +110,6 @@ FILTER Description >>>>>>>>>>> -Version: 3.1.0 - Usage: ``filter(array, function)`` filter the element in the array by the lambda function. The function should return boolean. The lambda function accepts one single input. Argument type: array:ARRAY, function:LAMBDA @@ -128,13 +118,13 @@ Return type: ARRAY Example:: - PPL> source=people | eval array = array(1, -2, 3), result = filter(array, x -> x > 0) | fields result | head 1 + os> source=people | eval array = array(1, -2, 3), result = filter(array, x -> x > 0) | fields result | head 1 fetched rows / total rows = 1/1 - +---------+ - | result | - |---------| - | [1, 3] | - +---------+ + +--------+ + | result | + |--------| + | [1,3] | + +--------+ TRANSFORM --------- @@ -142,8 +132,6 @@ TRANSFORM Description >>>>>>>>>>> -Version: 3.1.0 - Usage: ``transform(array, function)`` transform the element of array one by one using lambda. The lambda function can accept one single input or two input. If the lambda accepts two argument, the second one is the index of element in array. Argument type: array:ARRAY, function:LAMBDA @@ -152,21 +140,21 @@ Return type: ARRAY Example:: - PPL> source=people | eval array = array(1, -2, 3), result = transform(array, x -> x + 2) | fields result | head 1 + os> source=people | eval array = array(1, -2, 3), result = transform(array, x -> x + 2) | fields result | head 1 fetched rows / total rows = 1/1 - +------------+ - | result | - |------------| - | [3, 0, 5] | - +------------+ + +---------+ + | result | + |---------| + | [3,0,5] | + +---------+ - PPL> source=people | eval array = array(1, -2, 3), result = transform(array, (x, i) -> x + i) | fields result | head 1 + os> source=people | eval array = array(1, -2, 3), result = transform(array, (x, i) -> x + i) | fields result | head 1 fetched rows / total rows = 1/1 - +------------+ - | result | - |------------| - | [1, -1, 5] | - +------------+ + +----------+ + | result | + |----------| + | [1,-1,5] | + +----------+ REDUCE ------ @@ -174,8 +162,6 @@ REDUCE Description >>>>>>>>>>> -Version: 3.1.0 - Usage: ``reduce(array, acc_base, function, )`` use lambda function to go through all element and interact with acc_base. The lambda function accept two argument accumulator and array element. If add one more reduce_function, will apply reduce_function to accumulator finally. The reduce function accept accumulator as the one argument. Argument type: array:ARRAY, acc_base:ANY, function:LAMBDA, reduce_function:LAMBDA @@ -184,21 +170,21 @@ Return type: ANY Example:: - PPL> source=people | eval array = array(1, -2, 3), result = reduce(array, 10, (acc, x) -> acc + x) | fields result | head 1 + os> source=people | eval array = array(1, -2, 3), result = reduce(array, 10, (acc, x) -> acc + x) | fields result | head 1 fetched rows / total rows = 1/1 - +------------+ - | result | - |------------| - | 8 | - +------------+ + +--------+ + | result | + |--------| + | 12 | + +--------+ - PPL> source=people | eval array = array(1, -2, 3), result = reduce(array, 10, (acc, x) -> acc + x, acc -> acc * 10) | fields result | head 1 + os> source=people | eval array = array(1, -2, 3), result = reduce(array, 10, (acc, x) -> acc + x, acc -> acc * 10) | fields result | head 1 fetched rows / total rows = 1/1 - +------------+ - | result | - |------------| - | 80 | - +------------+ + +--------+ + | result | + |--------| + | 120 | + +--------+ MVJOIN ------ @@ -206,8 +192,6 @@ MVJOIN Description >>>>>>>>>>> -Version: 3.3.0 - Usage: mvjoin(array, delimiter) joins string array elements into a single string, separated by the specified delimiter. NULL elements are excluded from the output. Only string arrays are supported. Argument type: array: ARRAY of STRING, delimiter: STRING @@ -216,19 +200,104 @@ Return type: STRING Example:: - PPL> source=people | eval result = mvjoin(array('a', 'b', 'c'), ',') | fields result | head 1 + os> source=people | eval result = mvjoin(array('a', 'b', 'c'), ',') | fields result | head 1 + fetched rows / total rows = 1/1 + +--------+ + | result | + |--------| + | a,b,c | + +--------+ + + os> source=accounts | eval names_array = array(firstname, lastname) | eval result = mvjoin(names_array, ', ') | fields result | head 1 + fetched rows / total rows = 1/1 + +-------------+ + | result | + |-------------| + | Amber, Duke | + +-------------+ + +MVAPPEND +-------- + +Description +>>>>>>>>>>> + +Usage: mvappend(value1, value2, value3...) appends all elements from arguments to create an array. Flattens array arguments and collects all individual elements. Always returns an array or null for consistent type behavior. + +Argument type: value1: ANY, value2: ANY, ... + +Return type: ARRAY + +Example:: + + os> source=people | eval result = mvappend(1, 1, 3) | fields result | head 1 + fetched rows / total rows = 1/1 + +---------+ + | result | + |---------| + | [1,1,3] | + +---------+ + + os> source=people | eval result = mvappend(1, array(2, 3)) | fields result | head 1 + fetched rows / total rows = 1/1 + +---------+ + | result | + |---------| + | [1,2,3] | + +---------+ + + os> source=people | eval result = mvappend(mvappend(1, 2), 3) | fields result | head 1 + fetched rows / total rows = 1/1 + +---------+ + | result | + |---------| + | [1,2,3] | + +---------+ + + os> source=people | eval result = mvappend(42) | fields result | head 1 + fetched rows / total rows = 1/1 + +--------+ + | result | + |--------| + | [42] | + +--------+ + + os> source=people | eval result = mvappend(nullif(1, 1), 2) | fields result | head 1 + fetched rows / total rows = 1/1 + +--------+ + | result | + |--------| + | [2] | + +--------+ + + os> source=people | eval result = mvappend(nullif(1, 1)) | fields result | head 1 + fetched rows / total rows = 1/1 + +--------+ + | result | + |--------| + | null | + +--------+ + + os> source=people | eval arr1 = array(1, 2), arr2 = array(3, 4), result = mvappend(arr1, arr2) | fields result | head 1 + fetched rows / total rows = 1/1 + +-----------+ + | result | + |-----------| + | [1,2,3,4] | + +-----------+ + + os> source=accounts | eval result = mvappend(firstname, lastname) | fields result | head 1 fetched rows / total rows = 1/1 - +------------------------------------+ - | result | - |------------------------------------| - | "a,b,c" | - +------------------------------------+ + +--------------+ + | result | + |--------------| + | [Amber,Duke] | + +--------------+ - PPL> source=accounts | eval names_array = array(firstname, lastname) | eval result = mvjoin(names_array, ', ') | fields result | head 1 + os> source=people | eval result = mvappend(1, 'text', 2.5) | fields result | head 1 fetched rows / total rows = 1/1 - +------------------------------------------+ - | result | - |------------------------------------------| - | "Amber, Duke" | - +------------------------------------------+ - + +--------------+ + | result | + |--------------| + | [1,text,2.5] | + +--------------+ diff --git a/docs/user/ppl/functions/condition.rst b/docs/user/ppl/functions/condition.rst index 6be77cd5f97..58feecdcfcb 100644 --- a/docs/user/ppl/functions/condition.rst +++ b/docs/user/ppl/functions/condition.rst @@ -14,9 +14,14 @@ ISNULL Description >>>>>>>>>>> -Usage: isnull(field) return true if field is null. +Usage: isnull(field) returns TRUE if field is NULL, FALSE otherwise. -Argument type: all the supported data type. +The `isnull()` function is commonly used: +- In `eval` expressions to create conditional fields +- With the `if()` function to provide default values +- In `where` clauses to filter null records + +Argument type: all the supported data types. Return type: BOOLEAN @@ -33,15 +38,44 @@ Example:: | True | null | Dale | +--------+----------+-----------+ +Using with if() to label records:: + + os> source=accounts | eval status = if(isnull(employer), 'unemployed', 'employed') | fields firstname, employer, status + fetched rows / total rows = 4/4 + +-----------+----------+------------+ + | firstname | employer | status | + |-----------+----------+------------| + | Amber | Pyrami | employed | + | Hattie | Netagy | employed | + | Nanette | Quility | employed | + | Dale | null | unemployed | + +-----------+----------+------------+ + +Filtering with where clause:: + + os> source=accounts | where isnull(employer) | fields account_number, firstname, employer + fetched rows / total rows = 1/1 + +----------------+-----------+----------+ + | account_number | firstname | employer | + |----------------+-----------+----------| + | 18 | Dale | null | + +----------------+-----------+----------+ + ISNOTNULL --------- Description >>>>>>>>>>> -Usage: isnotnull(field) return true if field is not null. +Usage: isnotnull(field) returns TRUE if field is NOT NULL, FALSE otherwise. -Argument type: all the supported data type. +The `isnotnull()` function is commonly used: +- In `eval` expressions to create boolean flags +- In `where` clauses to filter out null values +- With the `if()` function for conditional logic +- To validate data presence + +Argument type: all the supported data types. Return type: BOOLEAN @@ -49,6 +83,19 @@ Synonyms: `ISPRESENT`_ Example:: + os> source=accounts | eval has_employer = isnotnull(employer) | fields firstname, employer, has_employer + fetched rows / total rows = 4/4 + +-----------+----------+--------------+ + | firstname | employer | has_employer | + |-----------+----------+--------------| + | Amber | Pyrami | True | + | Hattie | Netagy | True | + | Nanette | Quility | True | + | Dale | null | False | + +-----------+----------+--------------+ + +Filtering with where clause:: + os> source=accounts | where not isnotnull(employer) | fields account_number, employer fetched rows / total rows = 1/1 +----------------+----------+ @@ -57,6 +104,19 @@ Example:: | 18 | null | +----------------+----------+ +Using with if() for validation messages:: + + os> source=accounts | eval validation = if(isnotnull(employer), 'valid', 'missing employer') | fields firstname, employer, validation + fetched rows / total rows = 4/4 + +-----------+----------+------------------+ + | firstname | employer | validation | + |-----------+----------+------------------| + | Amber | Pyrami | valid | + | Hattie | Netagy | valid | + | Nanette | Quility | valid | + | Dale | null | missing employer | + +-----------+----------+------------------+ + EXISTS ------ @@ -142,32 +202,6 @@ Example:: | null | null | Dale | +---------+----------+-----------+ - -ISNULL ------- - -Description ->>>>>>>>>>> - -Usage: isnull(field1, field2) return null if two parameters are same, otherwise return field1. - -Argument type: all the supported data type - -Return type: any - -Example:: - - os> source=accounts | eval result = isnull(employer) | fields result, employer, firstname - fetched rows / total rows = 4/4 - +--------+----------+-----------+ - | result | employer | firstname | - |--------+----------+-----------| - | False | Pyrami | Amber | - | False | Netagy | Hattie | - | False | Quility | Nanette | - | True | null | Dale | - +--------+----------+-----------+ - IF ------ @@ -227,6 +261,14 @@ Argument type: all the supported data type, (NOTE : there is no comma before "el Return type: any +Limitations +>>>>>>>>>>> + +When each condition is a field comparison with a numeric literal and each result expression is a string literal, the query will be optimized as `range aggregations `_ if pushdown optimization is enabled. However, this optimization has the following limitations: + +- Null values will not be grouped into any bucket of a range aggregation and will be ignored +- The default ELSE clause will use the string literal ``"null"`` instead of actual NULL values + Example:: os> source=accounts | eval result = case(age > 35, firstname, age < 30, lastname else employer) | fields result, firstname, lastname, age, employer diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index dbe4403540c..849d2334e41 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -46,7 +46,7 @@ Cast to string example:: +-------+------+------------+ | cbool | cint | cdate | |-------+------+------------| - | true | 1 | 2012-08-07 | + | TRUE | 1 | 2012-08-07 | +-------+------+------------+ Cast to number example:: @@ -78,3 +78,42 @@ Cast function can be chained:: |-------| | True | +-------+ + + +IMPLICIT (AUTO) TYPE CONVERSION +------------------------------- + +Implicit conversion is automatic casting. When a function does not have an exact match for the +input types, the engine looks for another signature that can safely work with the values. It picks +the option that requires the least stretching of the original types, so you can mix literals and +fields without adding ``CAST`` everywhere. + +String to numeric +>>>>>>>>>>>>>>>>> + +When a string stands in for a number we simply parse the text: + +- The value must be something like ``"3.14"`` or ``"42"``. Anything else causes the query to fail. +- If a string appears next to numeric arguments, it is treated as a ``DOUBLE`` so the numeric + overload of the function can run. + +Use string in arithmetic operator example :: + + os> source=people | eval divide="5"/10, multiply="5" * 10, add="5" + 10, minus="5" - 10, concat="5" + "5" | fields divide, multiply, add, minus, concat + fetched rows / total rows = 1/1 + +--------+----------+------+-------+--------+ + | divide | multiply | add | minus | concat | + |--------+----------+------+-------+--------| + | 0.5 | 50.0 | 15.0 | -5.0 | 55 | + +--------+----------+------+-------+--------+ + +Use string in comparison operator example :: + + os> source=people | eval e="1000"==1000, en="1000"!=1000, ed="1000"==1000.0, edn="1000"!=1000.0, l="1000">999, ld="1000">999.9, i="malformed"==1000 | fields e, en, ed, edn, l, ld, i + fetched rows / total rows = 1/1 + +------+-------+------+-------+------+------+------+ + | e | en | ed | edn | l | ld | i | + |------+-------+------+-------+------+------+------| + | True | False | True | False | True | True | null | + +------+-------+------+-------+------+------+------+ + diff --git a/docs/user/ppl/functions/expressions.rst b/docs/user/ppl/functions/expressions.rst index 5e10c3d4dd4..2b30c739a45 100644 --- a/docs/user/ppl/functions/expressions.rst +++ b/docs/user/ppl/functions/expressions.rst @@ -28,7 +28,10 @@ Arithmetic expression is an expression formed by numeric literals and binary ari 1. ``+``: Add. 2. ``-``: Subtract. 3. ``*``: Multiply. -4. ``/``: Divide. For integers, the result is an integer with fractional part discarded. Returns NULL when dividing by zero. +4. ``/``: Divide. Integer operands follow the legacy truncating result when + `plugins.ppl.syntax.legacy.preferred <../admin/settings.rst>`_ is ``true`` (default). When the + setting is ``false`` the operands are promoted to floating point, preserving + the fractional part. Division by zero still returns ``NULL``. 5. ``%``: Modulo. This can be used with integers only with remainder of the division as result. Precedence @@ -172,4 +175,3 @@ NOT operator :: | 36 | | 28 | +-----+ - diff --git a/docs/user/ppl/functions/ip.rst b/docs/user/ppl/functions/ip.rst index 897d6ccfad0..ec853c27093 100644 --- a/docs/user/ppl/functions/ip.rst +++ b/docs/user/ppl/functions/ip.rst @@ -45,7 +45,7 @@ Description Usage: `geoip(dataSourceName, ipAddress[, options])` to lookup location information from given IP addresses via OpenSearch GeoSpatial plugin API. -Argument type: STRING, STRING, STRING +Argument type: STRING, STRING/IP, STRING Return type: OBJECT diff --git a/docs/user/ppl/functions/string.rst b/docs/user/ppl/functions/string.rst index d0d38d8c72f..eb82a06a055 100644 --- a/docs/user/ppl/functions/string.rst +++ b/docs/user/ppl/functions/string.rst @@ -207,9 +207,23 @@ REPLACE Description >>>>>>>>>>> -Usage: replace(str, substr, newstr) returns a string with all occurrences of substr replaced by newstr in str. If any argument is NULL, the function returns NULL. +Usage: replace(str, pattern, replacement) returns a string with all occurrences of the pattern replaced by the replacement string in str. If any argument is NULL, the function returns NULL. -Example:: +**Regular Expression Support**: The pattern argument supports Java regex syntax, including: + +Argument type: STRING, STRING (regex pattern), STRING (replacement) + +Return type: STRING + +**Important - Regex Special Characters**: The pattern is interpreted as a regular expression. Characters like ``.``, ``*``, ``+``, ``[``, ``]``, ``(``, ``)``, ``{``, ``}``, ``^``, ``$``, ``|``, ``?``, and ``\`` have special meaning in regex. To match them literally, escape with backslashes: + +* To match ``example.com``: use ``'example\\.com'`` (escape the dots) +* To match ``value*``: use ``'value\\*'`` (escape the asterisk) +* To match ``price+tax``: use ``'price\\+tax'`` (escape the plus) + +For strings with many special characters, use ``\\Q...\\E`` to quote the entire literal string (e.g., ``'\\Qhttps://example.com/path?id=123\\E'`` matches that exact URL). + +Literal String Replacement Examples:: os> source=people | eval `REPLACE('helloworld', 'world', 'universe')` = REPLACE('helloworld', 'world', 'universe'), `REPLACE('helloworld', 'invalid', 'universe')` = REPLACE('helloworld', 'invalid', 'universe') | fields `REPLACE('helloworld', 'world', 'universe')`, `REPLACE('helloworld', 'invalid', 'universe')` fetched rows / total rows = 1/1 @@ -219,6 +233,51 @@ Example:: | hellouniverse | helloworld | +--------------------------------------------+----------------------------------------------+ +Escaping Special Characters Examples:: + + os> source=people | eval `Replace domain` = REPLACE('api.example.com', 'example\\.com', 'newsite.org'), `Replace with quote` = REPLACE('https://api.example.com/v1', '\\Qhttps://api.example.com\\E', 'http://localhost:8080') | fields `Replace domain`, `Replace with quote` + fetched rows / total rows = 1/1 + +-----------------+--------------------------+ + | Replace domain | Replace with quote | + |-----------------+--------------------------| + | api.newsite.org | http://localhost:8080/v1 | + +-----------------+--------------------------+ + +Regex Pattern Examples:: + + os> source=people | eval `Remove digits` = REPLACE('test123', '\\d+', ''), `Collapse spaces` = REPLACE('hello world', ' +', ' '), `Remove special` = REPLACE('hello@world!', '[^a-zA-Z]', '') | fields `Remove digits`, `Collapse spaces`, `Remove special` + fetched rows / total rows = 1/1 + +---------------+-----------------+----------------+ + | Remove digits | Collapse spaces | Remove special | + |---------------+-----------------+----------------| + | test | hello world | helloworld | + +---------------+-----------------+----------------+ + +Capture Group and Backreference Examples:: + + os> source=people | eval `Swap date` = REPLACE('1/14/2023', '^(\\d{1,2})/(\\d{1,2})/', '$2/$1/'), `Reverse words` = REPLACE('Hello World', '(\\w+) (\\w+)', '$2 $1'), `Extract domain` = REPLACE('user@example.com', '.*@(.+)', '$1') | fields `Swap date`, `Reverse words`, `Extract domain` + fetched rows / total rows = 1/1 + +-----------+---------------+----------------+ + | Swap date | Reverse words | Extract domain | + |-----------+---------------+----------------| + | 14/1/2023 | World Hello | example.com | + +-----------+---------------+----------------+ + +Advanced Regex Examples:: + + os> source=people | eval `Clean phone` = REPLACE('(555) 123-4567', '[^0-9]', ''), `Remove vowels` = REPLACE('hello world', '[aeiou]', ''), `Add prefix` = REPLACE('test', '^', 'pre_') | fields `Clean phone`, `Remove vowels`, `Add prefix` + fetched rows / total rows = 1/1 + +-------------+---------------+------------+ + | Clean phone | Remove vowels | Add prefix | + |-------------+---------------+------------| + | 5551234567 | hll wrld | pre_test | + +-------------+---------------+------------+ + +**Note**: When using regex patterns in PPL queries: + +* Backslashes must be escaped (use ``\\`` instead of ``\``) - e.g., ``\\d`` for digit pattern, ``\\w+`` for word characters +* Backreferences support both PCRE-style (``\1``, ``\2``, etc.) and Java-style (``$1``, ``$2``, etc.) syntax. PCRE-style backreferences are automatically converted to Java-style internally. + REVERSE ------- diff --git a/docs/user/ppl/index.rst b/docs/user/ppl/index.rst index 7f329d28773..697ec7e2c6e 100644 --- a/docs/user/ppl/index.rst +++ b/docs/user/ppl/index.rst @@ -88,6 +88,8 @@ The query start with search command and then flowing a set of command delimited - `ml command `_ + - `multisearch command `_ + - `parse command `_ - `patterns command `_ @@ -110,6 +112,8 @@ The query start with search command and then flowing a set of command delimited - `stats command `_ + - `streamstats command `_ + - `subquery (aka subsearch) command `_ - `reverse command `_ @@ -122,6 +126,8 @@ The query start with search command and then flowing a set of command delimited - `trendline command `_ + - `replace command `_ + - `where command `_ * **Functions** diff --git a/docs/user/ppl/interfaces/endpoint.rst b/docs/user/ppl/interfaces/endpoint.rst index 967761caa37..b4acc21d8f4 100644 --- a/docs/user/ppl/interfaces/endpoint.rst +++ b/docs/user/ppl/interfaces/endpoint.rst @@ -73,28 +73,78 @@ Description You can send HTTP explain request to endpoint **/_plugins/_ppl/_explain** with your query in request body to understand the execution plan for the PPL query. The explain endpoint is useful when user want to get insight how the query is executed in the engine. -Example -------- +Description +----------- + +To translate your query, send it to explain endpoint. The explain output is OpenSearch domain specific language (DSL) in JSON format. You can just copy and paste it to your console to run it against OpenSearch directly. + +Explain output could be set different formats: ``standard`` (the default format), ``simple``, ``extended``, ``dsl``. + + +Example 1 default (standard) format +----------------------------------- -The following PPL query demonstrated that where and stats command were pushed down to OpenSearch DSL aggregation query:: +Explain query:: sh$ curl -sS -H 'Content-Type: application/json' \ ... -X POST localhost:9200/_plugins/_ppl/_explain \ - ... -d '{"query" : "source=accounts | where age > 10 | stats avg(age)"}' + ... -d '{"query" : "source=state_country | where age>30"}' { - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[avg(age)]" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":10,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}, searchDone=false)" - }, - "children": [] - } - ] + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(name=[$0], country=[$1], state=[$2], month=[$3], year=[$4], age=[$5])\n LogicalFilter(condition=[>($5, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, state_country]])\n", + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, state_country]], PushDownContext=[[PROJECT->[name, country, state, month, year, age], FILTER->>($5, 30), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"name\",\"country\",\"state\",\"month\",\"year\",\"age\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" + } + } + +Example 2 simple format +----------------------- + +Explain query:: + + sh$ curl -sS -H 'Content-Type: application/json' \ + ... -X POST localhost:9200/_plugins/_ppl/_explain?format=simple \ + ... -d '{"query" : "source=state_country | where age>30"}' + { + "calcite": { + "logical": "LogicalSystemLimit\n LogicalProject\n LogicalFilter\n CalciteLogicalIndexScan\n" } } + +Example 3 extended format +------------------------- + +Explain query:: + + sh$ curl -sS -H 'Content-Type: application/json' \ + ... -X POST localhost:9200/_plugins/_ppl/_explain?format=extended \ + ... -d '{"query" : "source=state_country | where age>30 | dedup age"}' + { + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(name=[$0], country=[$1], state=[$2], month=[$3], year=[$4], age=[$5])\n LogicalFilter(condition=[<=($12, 1)])\n LogicalProject(name=[$0], country=[$1], state=[$2], month=[$3], year=[$4], age=[$5], _id=[$6], _index=[$7], _score=[$8], _maxscore=[$9], _sort=[$10], _routing=[$11], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $5 ORDER BY $5)])\n LogicalFilter(condition=[IS NOT NULL($5)])\n LogicalFilter(condition=[>($5, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, state_country]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..6=[{inputs}], expr#7=[1], expr#8=[<=($t6, $t7)], proj#0..5=[{exprs}], $condition=[$t8])\n EnumerableWindow(window#0=[window(partition {5} order by [5] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, state_country]], PushDownContext=[[PROJECT->[name, country, state, month, year, age], FILTER->>($5, 30)], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"name\",\"country\",\"state\",\"month\",\"year\",\"age\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n", + "extended": "public org.apache.calcite.linq4j.Enumerable bind(final org.apache.calcite.DataContext root) {\n final org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan v1stashed = (org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan) root.get(\"v1stashed\");\n int prevStart;\n int prevEnd;\n final java.util.Comparator comparator = new java.util.Comparator(){\n public int compare(Object[] v0, Object[] v1) {\n final int c;\n c = org.apache.calcite.runtime.Utilities.compareNullsLast((Long) v0[5], (Long) v1[5]);\n if (c != 0) {\n return c;\n }\n return 0;\n }\n\n public int compare(Object o0, Object o1) {\n return this.compare((Object[]) o0, (Object[]) o1);\n }\n\n };\n final org.apache.calcite.runtime.SortedMultiMap multiMap = new org.apache.calcite.runtime.SortedMultiMap();\n v1stashed.scan().foreach(new org.apache.calcite.linq4j.function.Function1() {\n public Object apply(Object[] v) {\n Long key = (Long) v[5];\n multiMap.putMulti(key, v);\n return null;\n }\n public Object apply(Object v) {\n return apply(\n (Object[]) v);\n }\n }\n );\n final java.util.Iterator iterator = multiMap.arrays(comparator);\n final java.util.ArrayList _list = new java.util.ArrayList(\n multiMap.size());\n Long a0w0 = (Long) null;\n while (iterator.hasNext()) {\n final Object[] _rows = (Object[]) iterator.next();\n prevStart = -1;\n prevEnd = 2147483647;\n for (int i = 0; i < _rows.length; (++i)) {\n final Object[] row = (Object[]) _rows[i];\n if (i != prevEnd) {\n int actualStart = i < prevEnd ? 0 : prevEnd + 1;\n prevEnd = i;\n a0w0 = Long.valueOf(((Number)org.apache.calcite.linq4j.tree.Primitive.of(long.class).numberValueRoundDown((i - 0 + 1))).longValue());\n }\n _list.add(new Object[] {\n row[0],\n row[1],\n row[2],\n row[3],\n row[4],\n row[5],\n a0w0});\n }\n }\n multiMap.clear();\n final org.apache.calcite.linq4j.Enumerable _inputEnumerable = org.apache.calcite.linq4j.Linq4j.asEnumerable(_list);\n final org.apache.calcite.linq4j.AbstractEnumerable child = new org.apache.calcite.linq4j.AbstractEnumerable(){\n public org.apache.calcite.linq4j.Enumerator enumerator() {\n return new org.apache.calcite.linq4j.Enumerator(){\n public final org.apache.calcite.linq4j.Enumerator inputEnumerator = _inputEnumerable.enumerator();\n public void reset() {\n inputEnumerator.reset();\n }\n\n public boolean moveNext() {\n while (inputEnumerator.moveNext()) {\n if (org.apache.calcite.runtime.SqlFunctions.toLong(((Object[]) inputEnumerator.current())[6]) <= $L4J$C$_Number_org_apache_calcite_linq4j_tree_Primitive_of_long_class_358aa52b) {\n return true;\n }\n }\n return false;\n }\n\n public void close() {\n inputEnumerator.close();\n }\n\n public Object current() {\n final Object[] current = (Object[]) inputEnumerator.current();\n final Object input_value = current[0];\n final Object input_value0 = current[1];\n final Object input_value1 = current[2];\n final Object input_value2 = current[3];\n final Object input_value3 = current[4];\n final Object input_value4 = current[5];\n return new Object[] {\n input_value,\n input_value0,\n input_value1,\n input_value2,\n input_value3,\n input_value4};\n }\n\n static final long $L4J$C$_Number_org_apache_calcite_linq4j_tree_Primitive_of_long_class_358aa52b = ((Number)org.apache.calcite.linq4j.tree.Primitive.of(long.class).numberValueRoundDown(1)).longValue();\n };\n }\n\n };\n return child.take(10000);\n}\n\n\npublic Class getElementType() {\n return java.lang.Object[].class;\n}\n\n\n" + } + } + +Example 4 YAML format (experimental) +----------------------------------- + +.. note:: + YAML explain output is an experimental feature and not intended for + production use. The interface and output may change without notice. + +Return Explain response format in In ``yaml`` format. + +Explain query:: + + sh$ curl -sS -H 'Content-Type: application/json' \ + ... -X POST localhost:9200/_plugins/_ppl/_explain?format=yaml \ + ... -d '{"query" : "source=state_country | where age>30"}' + calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(name=[$0], country=[$1], state=[$2], month=[$3], year=[$4], age=[$5]) + LogicalFilter(condition=[>($5, 30)]) + CalciteLogicalIndexScan(table=[[OpenSearch, state_country]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, state_country]], PushDownContext=[[PROJECT->[name, country, state, month, year, age], FILTER->>($5, 30), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"_source":{"includes":["name","country","state","month","year","age"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/doctest/build.gradle b/doctest/build.gradle index d3492b62917..d758a6de4b8 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -19,6 +19,8 @@ def path = project(':').projectDir // temporary fix, because currently we are under migration to new architecture. Need to run ./gradlew run from // plugin module, and will only build ppl in it. def plugin_path = project(':doctest').projectDir +String ignorePrometheusProp = System.getProperty("ignorePrometheus") +boolean ignorePrometheus = ignorePrometheusProp != null && !ignorePrometheusProp.equalsIgnoreCase("false") task cloneSqlCli(type: Exec) { def repoDir = new File("${project.projectDir}/sql-cli") @@ -41,6 +43,19 @@ task bootstrap(type: Exec, dependsOn: ['cloneSqlCli']) { } +def isPrometheusRunning = { -> + try { + def process = "pgrep -f prometheus".execute() + def output = process.text + def result = !output.trim().isEmpty() + println "Prometheus running status: ${result}" + return result + } catch (Exception e) { + println "Error checking Prometheus process: ${e.message}" + return false + } +} + task startPrometheus(type: SpawnProcessTask) { doFirst { download.run { @@ -61,6 +76,7 @@ task startPrometheus(type: SpawnProcessTask) { command "$projectDir/bin/prometheus/prometheus --storage.tsdb.path=$projectDir/bin/prometheus/data --config.file=$projectDir/bin/prometheus/prometheus.yml" ready 'TSDB started' pidLockFileName ".prom.pid.lock" + onlyIf { !ignorePrometheus && getOSFamilyType() != "windows" && !isPrometheusRunning() } } //evaluationDependsOn(':') @@ -88,6 +104,7 @@ task doctest(type: Exec, dependsOn: ['bootstrap']) { if (debug == 'true') { environment 'DOCTEST_DEBUG', 'true' } + environment 'IGNORE_PROMETHEUS_DOCS', ignorePrometheus ? 'true' : 'false' if (docs) { def args = ['.venv/bin/python', 'test_docs.py'] @@ -121,14 +138,17 @@ task stopPrometheus(type: KillProcessTask) { file("$projectDir/bin/prometheus").deleteDir() file("$projectDir/bin/prometheus.tar.gz").delete() } + onlyIf { !ignorePrometheus && getOSFamilyType() != "windows" } } // Stop Prom AFTER Start Prom... if(getOSFamilyType() != "windows") { stopPrometheus.mustRunAfter startPrometheus - startOpenSearch.dependsOn startPrometheus - stopOpenSearch.finalizedBy stopPrometheus - startOpenSearch.finalizedBy stopPrometheus + if (!ignorePrometheus) { + startOpenSearch.dependsOn startPrometheus + stopOpenSearch.finalizedBy stopPrometheus + startOpenSearch.finalizedBy stopPrometheus + } } doctest.dependsOn startOpenSearch doctest.finalizedBy stopOpenSearch diff --git a/doctest/test_data/events.json b/doctest/test_data/events.json index e873691fb90..ea63088151a 100644 --- a/doctest/test_data/events.json +++ b/doctest/test_data/events.json @@ -1,8 +1,8 @@ -{"@timestamp":"2023-01-01T10:00:00Z","event_time":"2023-01-01T09:55:00Z","host":"server1","message":"Starting up","level":"INFO","category":"orders","status":"pending"} -{"@timestamp":"2023-01-01T10:05:00Z","event_time":"2023-01-01T10:00:00Z","host":"server2","message":"Initializing","level":"INFO","category":"users","status":"active"} -{"@timestamp":"2023-01-01T10:10:00Z","event_time":"2023-01-01T10:05:00Z","host":"server1","message":"Ready to serve","level":"INFO","category":"orders","status":"processing"} -{"@timestamp":"2023-01-01T10:15:00Z","event_time":"2023-01-01T10:10:00Z","host":"server2","message":"Ready","level":"INFO","category":"users","status":"inactive"} -{"@timestamp":"2023-01-01T10:20:00Z","event_time":"2023-01-01T10:15:00Z","host":"server1","message":"Processing requests","level":"INFO","category":"orders","status":"completed"} -{"@timestamp":"2023-01-01T10:25:00Z","event_time":"2023-01-01T10:20:00Z","host":"server2","message":"Handling connections","level":"INFO","category":"users","status":"pending"} -{"@timestamp":"2023-01-01T10:30:00Z","event_time":"2023-01-01T10:25:00Z","host":"server1","message":"Shutting down","level":"WARN","category":"orders","status":"cancelled"} -{"@timestamp":"2023-01-01T10:35:00Z","event_time":"2023-01-01T10:30:00Z","host":"server2","message":"Maintenance mode","level":"WARN","category":"users","status":"inactive"} +{"@timestamp":"2023-01-01T10:00:00Z","event_time":"2023-01-01T09:55:00Z","host":"server1","message":"Starting up","level":"INFO","category":"orders","status":"pending","packets":60} +{"@timestamp":"2023-01-01T10:05:00Z","event_time":"2023-01-01T10:00:00Z","host":"server2","message":"Initializing","level":"INFO","category":"users","status":"active","packets":30} +{"@timestamp":"2023-01-01T10:10:00Z","event_time":"2023-01-01T10:05:00Z","host":"server1","message":"Ready to serve","level":"INFO","category":"orders","status":"processing","packets":60} +{"@timestamp":"2023-01-01T10:15:00Z","event_time":"2023-01-01T10:10:00Z","host":"server2","message":"Ready","level":"INFO","category":"users","status":"inactive","packets":30} +{"@timestamp":"2023-01-01T10:20:00Z","event_time":"2023-01-01T10:15:00Z","host":"server1","message":"Processing requests","level":"INFO","category":"orders","status":"completed","packets":60} +{"@timestamp":"2023-01-01T10:25:00Z","event_time":"2023-01-01T10:20:00Z","host":"server2","message":"Handling connections","level":"INFO","category":"users","status":"pending","packets":30} +{"@timestamp":"2023-01-01T10:30:00Z","event_time":"2023-01-01T10:25:00Z","host":"server1","message":"Shutting down","level":"WARN","category":"orders","status":"cancelled","packets":180} +{"@timestamp":"2023-01-01T10:35:00Z","event_time":"2023-01-01T10:30:00Z","host":"server2","message":"Maintenance mode","level":"WARN","category":"users","status":"inactive","packets":90} diff --git a/doctest/test_data/time_test_data.json b/doctest/test_data/time_test_data.json new file mode 100644 index 00000000000..841beb04700 --- /dev/null +++ b/doctest/test_data/time_test_data.json @@ -0,0 +1,40 @@ +{"index":{"_id":"1"}} +{"timestamp":"2025-07-31T08:58:54","value":6795,"category":"D","@timestamp":"2025-07-31T08:58:54"} +{"index":{"_id":"2"}} +{"timestamp":"2025-07-31T09:45:39","value":8571,"category":"B","@timestamp":"2025-07-31T09:45:39"} +{"index":{"_id":"3"}} +{"timestamp":"2025-07-31T10:32:24","value":7238,"category":"C","@timestamp":"2025-07-31T10:32:24"} +{"index":{"_id":"4"}} +{"timestamp":"2025-07-31T11:19:09","value":8914,"category":"A","@timestamp":"2025-07-31T11:19:09"} +{"index":{"_id":"5"}} +{"timestamp":"2025-07-31T12:06:02","value":6580,"category":"D","@timestamp":"2025-07-31T12:06:02"} +{"index":{"_id":"6"}} +{"timestamp":"2025-07-31T13:52:47","value":9102,"category":"B","@timestamp":"2025-07-31T13:52:47"} +{"index":{"_id":"7"}} +{"timestamp":"2025-07-31T14:39:32","value":7425,"category":"C","@timestamp":"2025-07-31T14:39:32"} +{"index":{"_id":"8"}} +{"timestamp":"2025-07-31T15:26:17","value":8753,"category":"A","@timestamp":"2025-07-31T15:26:17"} +{"index":{"_id":"9"}} +{"timestamp":"2025-07-31T16:13:10","value":6338,"category":"D","@timestamp":"2025-07-31T16:13:10"} +{"index":{"_id":"10"}} +{"timestamp":"2025-07-31T17:59:55","value":8909,"category":"B","@timestamp":"2025-07-31T17:59:55"} +{"index":{"_id":"11"}} +{"timestamp":"2025-07-31T18:46:40","value":7682,"category":"C","@timestamp":"2025-07-31T18:46:40"} +{"index":{"_id":"12"}} +{"timestamp":"2025-07-31T19:33:25","value":9231,"category":"A","@timestamp":"2025-07-31T19:33:25"} +{"index":{"_id":"13"}} +{"timestamp":"2025-07-31T20:20:18","value":6824,"category":"D","@timestamp":"2025-07-31T20:20:18"} +{"index":{"_id":"14"}} +{"timestamp":"2025-07-31T21:07:03","value":8490,"category":"B","@timestamp":"2025-07-31T21:07:03"} +{"index":{"_id":"15"}} +{"timestamp":"2025-07-31T22:53:48","value":7153,"category":"C","@timestamp":"2025-07-31T22:53:48"} +{"index":{"_id":"16"}} +{"timestamp":"2025-07-31T23:40:33","value":8676,"category":"A","@timestamp":"2025-07-31T23:40:33"} +{"index":{"_id":"17"}} +{"timestamp":"2025-08-01T00:27:26","value":6489,"category":"D","@timestamp":"2025-08-01T00:27:26"} +{"index":{"_id":"18"}} +{"timestamp":"2025-08-01T01:14:11","value":9015,"category":"B","@timestamp":"2025-08-01T01:14:11"} +{"index":{"_id":"19"}} +{"timestamp":"2025-08-01T02:00:56","value":7348,"category":"C","@timestamp":"2025-08-01T02:00:56"} +{"index":{"_id":"20"}} +{"timestamp":"2025-08-01T03:47:41","value":8762,"category":"A","@timestamp":"2025-08-01T03:47:41"} diff --git a/doctest/test_data/time_test_data2.json b/doctest/test_data/time_test_data2.json new file mode 100644 index 00000000000..db92260798c --- /dev/null +++ b/doctest/test_data/time_test_data2.json @@ -0,0 +1,40 @@ +{"index":{"_id":"1"}} +{"timestamp":"2025-08-01T04:00:00","value":2001,"category":"E","@timestamp":"2025-08-01T04:00:00"} +{"index":{"_id":"2"}} +{"timestamp":"2025-08-01T02:30:00","value":2002,"category":"F","@timestamp":"2025-08-01T02:30:00"} +{"index":{"_id":"3"}} +{"timestamp":"2025-08-01T01:00:00","value":2003,"category":"E","@timestamp":"2025-08-01T01:00:00"} +{"index":{"_id":"4"}} +{"timestamp":"2025-07-31T22:15:00","value":2004,"category":"F","@timestamp":"2025-07-31T22:15:00"} +{"index":{"_id":"5"}} +{"timestamp":"2025-07-31T20:45:00","value":2005,"category":"E","@timestamp":"2025-07-31T20:45:00"} +{"index":{"_id":"6"}} +{"timestamp":"2025-07-31T18:30:00","value":2006,"category":"F","@timestamp":"2025-07-31T18:30:00"} +{"index":{"_id":"7"}} +{"timestamp":"2025-07-31T16:00:00","value":2007,"category":"E","@timestamp":"2025-07-31T16:00:00"} +{"index":{"_id":"8"}} +{"timestamp":"2025-07-31T14:15:00","value":2008,"category":"F","@timestamp":"2025-07-31T14:15:00"} +{"index":{"_id":"9"}} +{"timestamp":"2025-07-31T12:30:00","value":2009,"category":"E","@timestamp":"2025-07-31T12:30:00"} +{"index":{"_id":"10"}} +{"timestamp":"2025-07-31T10:45:00","value":2010,"category":"F","@timestamp":"2025-07-31T10:45:00"} +{"index":{"_id":"11"}} +{"timestamp":"2025-07-31T08:00:00","value":2011,"category":"E","@timestamp":"2025-07-31T08:00:00"} +{"index":{"_id":"12"}} +{"timestamp":"2025-07-31T06:15:00","value":2012,"category":"F","@timestamp":"2025-07-31T06:15:00"} +{"index":{"_id":"13"}} +{"timestamp":"2025-07-31T04:30:00","value":2013,"category":"E","@timestamp":"2025-07-31T04:30:00"} +{"index":{"_id":"14"}} +{"timestamp":"2025-07-31T02:45:00","value":2014,"category":"F","@timestamp":"2025-07-31T02:45:00"} +{"index":{"_id":"15"}} +{"timestamp":"2025-07-31T01:00:00","value":2015,"category":"E","@timestamp":"2025-07-31T01:00:00"} +{"index":{"_id":"16"}} +{"timestamp":"2025-07-30T23:15:00","value":2016,"category":"F","@timestamp":"2025-07-30T23:15:00"} +{"index":{"_id":"17"}} +{"timestamp":"2025-07-30T21:30:00","value":2017,"category":"E","@timestamp":"2025-07-30T21:30:00"} +{"index":{"_id":"18"}} +{"timestamp":"2025-07-30T19:45:00","value":2018,"category":"F","@timestamp":"2025-07-30T19:45:00"} +{"index":{"_id":"19"}} +{"timestamp":"2025-07-30T18:00:00","value":2019,"category":"E","@timestamp":"2025-07-30T18:00:00"} +{"index":{"_id":"20"}} +{"timestamp":"2025-07-30T16:15:00","value":2020,"category":"F","@timestamp":"2025-07-30T16:15:00"} diff --git a/doctest/test_docs.py b/doctest/test_docs.py index c94378ed537..c2d1112b584 100644 --- a/doctest/test_docs.py +++ b/doctest/test_docs.py @@ -42,10 +42,16 @@ 'work_information': 'work_information.json', 'events': 'events.json', 'otellogs': 'otellogs.json', + 'time_data': 'time_test_data.json', + 'time_data2': 'time_test_data2.json', 'time_test': 'time_test.json' } DEBUG_MODE = os.environ.get('DOCTEST_DEBUG', 'false').lower() == 'true' +IGNORE_PROMETHEUS_DOCS = os.environ.get('IGNORE_PROMETHEUS_DOCS', 'false').lower() == 'true' +PROMETHEUS_DOC_FILES = { + 'user/ppl/cmd/showdatasources.rst' +} def debug(message): @@ -100,6 +106,13 @@ def load_categories(self, file_path): try: with open(file_path) as json_file: categories = json.load(json_file) + if IGNORE_PROMETHEUS_DOCS: + categories = { + category: [ + doc for doc in docs if doc not in PROMETHEUS_DOC_FILES + ] + for category, docs in categories.items() + } debug(f"Loaded {len(categories)} categories from {file_path}") return categories except Exception as e: @@ -352,6 +365,7 @@ def create_cli_suite(filepaths, parser, setup_func): # Entry point for unittest discovery def load_tests(loader, suite, ignore): tests = [] + settings_tests = [] category_manager = CategoryManager() for category_name in category_manager.get_all_categories(): @@ -359,9 +373,16 @@ def load_tests(loader, suite, ignore): if not docs: continue - tests.append(get_test_suite(category_manager, category_name, get_doc_filepaths(docs))) + suite = get_test_suite(category_manager, category_name, get_doc_filepaths(docs)) + if 'settings' in category_name: + settings_tests.append(suite) + else: + tests.append(suite) random.shuffle(tests) + if settings_tests: + random.shuffle(settings_tests) + tests.extend(settings_tests) return DocTests(tests) def get_test_suite(category_manager: CategoryManager, category_name, filepaths): diff --git a/doctest/test_mapping/events.json b/doctest/test_mapping/events.json index 664f042324b..2c405a23fbb 100644 --- a/doctest/test_mapping/events.json +++ b/doctest/test_mapping/events.json @@ -23,6 +23,9 @@ }, "status": { "type": "keyword" + }, + "packets": { + "type": "integer" } } } diff --git a/gradle.properties b/gradle.properties index cca553e80bf..d95eee2f1a1 100644 --- a/gradle.properties +++ b/gradle.properties @@ -5,3 +5,4 @@ version=1.13.0 org.gradle.jvmargs=-Duser.language=en -Duser.country=US org.gradle.parallel=true +org.gradle.caching=true diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 61b6c536b7c..b6c5c0cd265 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -53,10 +53,16 @@ String bwcVersion = baseVersion + ".0"; String baseName = "sqlBwcCluster" String bwcFilePath = "src/test/resources/bwc/" String calciteCodegen = "$projectDir/src/test/java/codegen/" +String ignorePrometheusProp = System.getProperty("ignorePrometheus") +boolean ignorePrometheus = ignorePrometheusProp != null && !ignorePrometheusProp.equalsIgnoreCase("false") repositories { + mavenLocal() mavenCentral() - maven { url 'https://jitpack.io' } + maven { + url 'https://jitpack.io' + content { includeGroup "com.github.babbel" } + } // Add extra repository for the JDBC driver if given by user if (System.getProperty("jdbcRepo") != null && new File(System.getProperty("jdbcRepo")).isDirectory()) { @@ -181,6 +187,7 @@ configurations.all { resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" resolutionStrategy.force "com.squareup.okhttp3:okhttp:4.12.0" resolutionStrategy.force "org.apache.httpcomponents:httpcore:4.4.13" + resolutionStrategy.force "org.apache.httpcomponents:httpclient:4.5.14" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10" resolutionStrategy.force "joda-time:joda-time:2.10.12" @@ -293,6 +300,19 @@ testClusters { } } +def isPrometheusRunning() { + try { + def process = "pgrep -f prometheus".execute() + def output = process.text + def result = !output.trim().isEmpty() + println "Prometheus running status: ${result}" + return result + } catch (Exception e) { + println "Error checking Prometheus process: ${e.message}" + return false + } +} + task startPrometheus(type: SpawnProcessTask) { mustRunAfter ':doctest:doctest' @@ -317,6 +337,7 @@ task startPrometheus(type: SpawnProcessTask) { } command "$projectDir/bin/prometheus/prometheus --storage.tsdb.path=$projectDir/bin/prometheus/data --config.file=$projectDir/bin/prometheus/prometheus.yml" ready 'TSDB started' + onlyIf { !ignorePrometheus && !isPrometheusRunning() } } task stopPrometheus(type: KillProcessTask) { @@ -459,7 +480,7 @@ integTest { } dependsOn ':opensearch-sql-plugin:bundlePlugin' - if(getOSFamilyType() != "windows") { + if(!ignorePrometheus && getOSFamilyType() != "windows") { dependsOn startPrometheus finalizedBy stopPrometheus } @@ -499,7 +520,7 @@ integTest { } } - if(getOSFamilyType() == "windows") { + if(getOSFamilyType() == "windows" || ignorePrometheus) { exclude 'org/opensearch/sql/ppl/PrometheusDataSourceCommandsIT.class' exclude 'org/opensearch/sql/ppl/ShowDataSourcesCommandIT.class' exclude 'org/opensearch/sql/ppl/InformationSchemaCommandIT.class' @@ -572,33 +593,30 @@ task comparisonTest(type: RestIntegTestTask) { systemProperty "queries", System.getProperty("queries") } -2.times { i -> - testClusters { - "${baseName}$i" { - testDistribution = "ARCHIVE" - versions = [baseVersion, opensearch_version] - numberOfNodes = 3 - plugin(provider { (RegularFile) (() -> { - if (new File("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion").exists()) { - project.delete(files("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion")) - } - project.mkdir bwcJobSchedulerPath + bwcVersion - ant.get(src: bwcOpenSearchJSDownload, - dest: bwcJobSchedulerPath + bwcVersion, - httpusecaches: false) - return fileTree(bwcJobSchedulerPath + bwcVersion).getSingleFile() - })}) - plugin(provider { (RegularFile) (() -> { - return configurations.zipArchive.asFileTree.matching { - include '**/opensearch-sql-plugin*' - }.singleFile - })}) - setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" - setting 'http.content_type.required', 'true' - } +testClusters { + "${baseName}" { + testDistribution = "ARCHIVE" + versions = [baseVersion, opensearch_version] + numberOfNodes = 3 + plugin(provider { (RegularFile) (() -> { + if (new File("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion").exists()) { + project.delete(files("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion")) + } + project.mkdir bwcJobSchedulerPath + bwcVersion + ant.get(src: bwcOpenSearchJSDownload, + dest: bwcJobSchedulerPath + bwcVersion, + httpusecaches: false) + return fileTree(bwcJobSchedulerPath + bwcVersion).getSingleFile() + })}) + plugin(provider { (RegularFile) (() -> { + return configurations.zipArchive.asFileTree.matching { + include '**/opensearch-sql-plugin*' + }.singleFile + })}) + setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" + setting 'http.content_type.required', 'true' } } - List> plugins = [ getJobSchedulerPlugin(), provider { (RegularFile) (() -> @@ -606,29 +624,27 @@ List> plugins = [ } ] -// Creates 2 test clusters with 3 nodes of the old version. -2.times { i -> - task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) { - useCluster testClusters."${baseName}$i" - filter { - includeTestsMatching "org.opensearch.sql.bwc.*IT" - } - systemProperty 'tests.rest.bwcsuite', 'old_cluster' - systemProperty 'tests.rest.bwcsuite_round', 'old' - systemProperty 'tests.plugin_bwc_version', bwcVersion - nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}$i".allHttpSocketURI.join(",")}") - nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}$i".getName()}") +// Creates test cluster with 3 nodes of the old version. +task "${baseName}#oldVersionClusterTask"(type: StandaloneRestIntegTestTask) { + useCluster testClusters."${baseName}" + filter { + includeTestsMatching "org.opensearch.sql.bwc.*IT" } + systemProperty 'tests.rest.bwcsuite', 'old_cluster' + systemProperty 'tests.rest.bwcsuite_round', 'old' + systemProperty 'tests.plugin_bwc_version', bwcVersion + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") } // Upgrade one node of the old cluster to new OpenSearch version with upgraded plugin version. // This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node. // This is also used as a one third upgraded cluster for a rolling upgrade. task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) { - useCluster testClusters."${baseName}0" - dependsOn "${baseName}#oldVersionClusterTask0" + useCluster testClusters."${baseName}" + dependsOn "${baseName}#oldVersionClusterTask" doFirst { - testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) + testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins) } filter { includeTestsMatching "org.opensearch.sql.bwc.*IT" @@ -636,8 +652,8 @@ task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) { systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' systemProperty 'tests.rest.bwcsuite_round', 'first' systemProperty 'tests.plugin_bwc_version', bwcVersion - nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}") - nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}") + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") } // Upgrade the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded. @@ -645,9 +661,9 @@ task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) { // This is used for rolling upgrade. task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) { dependsOn "${baseName}#mixedClusterTask" - useCluster testClusters."${baseName}0" + useCluster testClusters."${baseName}" doFirst { - testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) + testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins) } filter { includeTestsMatching "org.opensearch.sql.bwc.*IT" @@ -655,8 +671,8 @@ task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTas systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' systemProperty 'tests.rest.bwcsuite_round', 'second' systemProperty 'tests.plugin_bwc_version', bwcVersion - nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}") - nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}") + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") } // Upgrade the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded. @@ -664,9 +680,9 @@ task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTas // This is used for rolling upgrade. task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) { dependsOn "${baseName}#twoThirdsUpgradedClusterTask" - useCluster testClusters."${baseName}0" + useCluster testClusters."${baseName}" doFirst { - testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) + testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins) } filter { includeTestsMatching "org.opensearch.sql.bwc.*IT" @@ -675,29 +691,29 @@ task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' systemProperty 'tests.rest.bwcsuite_round', 'third' systemProperty 'tests.plugin_bwc_version', bwcVersion - nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}") - nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}") + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") } // Upgrade all the nodes of the old cluster to new OpenSearch version with upgraded plugin version // at the same time resulting in a fully upgraded cluster. task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) { - dependsOn "${baseName}#oldVersionClusterTask1" - useCluster testClusters."${baseName}1" + dependsOn "${baseName}#oldVersionClusterTask" + useCluster testClusters."${baseName}" doFirst { - testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins) + testClusters."${baseName}".upgradeAllNodesAndPluginsToNextVersion(plugins) } filter { includeTestsMatching "org.opensearch.sql.bwc.*IT" } systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster' systemProperty 'tests.plugin_bwc_version', bwcVersion - nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}1".allHttpSocketURI.join(",")}") - nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}1".getName()}") + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") } -// A bwc test suite which runs all the bwc tasks combined -task bwcTestSuite(type: StandaloneRestIntegTestTask) { +// A bwc test suite which runs all the bwc tasks in rolling upgrade +task bwcTestRollingUpgradeSuite(type: StandaloneRestIntegTestTask) { testLogging { events "passed", "skipped", "failed" } @@ -705,6 +721,15 @@ task bwcTestSuite(type: StandaloneRestIntegTestTask) { exclude '**/*IT*' dependsOn tasks.named("${baseName}#mixedClusterTask") dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask") +} + +// A bwc test suite which runs all the bwc tasks in full restart +task bwcTestFullRestartSuite(type: StandaloneRestIntegTestTask) { + testLogging { + events "passed", "skipped", "failed" + } + exclude '**/*Test*' + exclude '**/*IT*' dependsOn tasks.named("${baseName}#fullRestartClusterTask") } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java index d94bea7be77..15051417db1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java @@ -46,6 +46,7 @@ CalciteLegacyAPICompatibilityIT.class, CalciteLikeQueryIT.class, CalciteMathematicalFunctionIT.class, + CalciteMultisearchCommandIT.class, CalciteMultiValueStatsIT.class, CalciteNewAddedCommandsIT.class, CalciteNowLikeFunctionIT.class, @@ -65,6 +66,7 @@ CalcitePPLCryptographicFunctionIT.class, CalcitePPLDedupIT.class, CalcitePPLEventstatsIT.class, + CalciteStreamstatsCommandIT.class, CalcitePPLExistsSubqueryIT.class, CalcitePPLExplainIT.class, CalcitePPLFillnullIT.class, @@ -88,6 +90,7 @@ CalciteRegexCommandIT.class, CalciteRexCommandIT.class, CalciteRenameCommandIT.class, + CalciteReplaceCommandIT.class, CalciteResourceMonitorIT.class, CalciteSearchCommandIT.class, CalciteSettingsIT.class, diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/big5/CalcitePPLBig5IT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/big5/CalcitePPLBig5IT.java index cc49a571778..665b3f0a874 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/big5/CalcitePPLBig5IT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/big5/CalcitePPLBig5IT.java @@ -21,24 +21,25 @@ public void init() throws Exception { @Test public void bin_bins() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/bin_bins.ppl")); + String ppl = sanitize(loadExpectedQuery("bin_bins.ppl")); timing(summary, "bin_bins", ppl); } @Test public void bin_span_log() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/bin_span_log.ppl")); + String ppl = sanitize(loadExpectedQuery("bin_span_log.ppl")); timing(summary, "bin_span_log", ppl); } @Test public void bin_span_time() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/bin_span_time.ppl")); + String ppl = sanitize(loadExpectedQuery("bin_span_time.ppl")); timing(summary, "bin_span_time", ppl); } + @Test public void coalesce_nonexistent_field_fallback() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/coalesce_nonexistent_field_fallback.ppl")); + String ppl = sanitize(loadExpectedQuery("coalesce_nonexistent_field_fallback.ppl")); timing(summary, "coalesce_nonexistent_field_fallback", ppl); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/big5/PPLBig5IT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/big5/PPLBig5IT.java index ae1e9881173..4997d361203 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/big5/PPLBig5IT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/big5/PPLBig5IT.java @@ -5,6 +5,8 @@ package org.opensearch.sql.calcite.big5; +import static org.opensearch.sql.util.MatcherUtils.assertYamlEqualsIgnoreId; + import java.io.IOException; import java.util.Locale; import java.util.Map; @@ -49,257 +51,383 @@ public static void reset() throws IOException { @Test public void asc_sort_timestamp() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/asc_sort_timestamp.ppl")); + String ppl = sanitize(loadExpectedQuery("asc_sort_timestamp.ppl")); timing(summary, "asc_sort_timestamp", ppl); + String expected = loadExpectedPlan("asc_sort_timestamp.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void asc_sort_timestamp_can_match_shortcut() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/asc_sort_timestamp_can_match_shortcut.ppl")); + String ppl = sanitize(loadExpectedQuery("asc_sort_timestamp_can_match_shortcut.ppl")); timing(summary, "asc_sort_timestamp_can_match_shortcut", ppl); + String expected = loadExpectedPlan("asc_sort_timestamp_can_match_shortcut.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void asc_sort_timestamp_no_can_match_shortcut() throws IOException { - String ppl = - sanitize(loadFromFile("big5/queries/asc_sort_timestamp_no_can_match_shortcut.ppl")); + String ppl = sanitize(loadExpectedQuery("asc_sort_timestamp_no_can_match_shortcut.ppl")); timing(summary, "asc_sort_timestamp_no_can_match_shortcut", ppl); + String expected = loadExpectedPlan("asc_sort_timestamp_no_can_match_shortcut.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void asc_sort_with_after_timestamp() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/asc_sort_with_after_timestamp.ppl")); + String ppl = sanitize(loadExpectedQuery("asc_sort_with_after_timestamp.ppl")); timing(summary, "asc_sort_with_after_timestamp", ppl); + String expected = loadExpectedPlan("asc_sort_with_after_timestamp.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void composite_date_histogram_daily() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/composite_date_histogram_daily.ppl")); + String ppl = sanitize(loadExpectedQuery("composite_date_histogram_daily.ppl")); timing(summary, "composite_date_histogram_daily", ppl); + String expected = loadExpectedPlan("composite_date_histogram_daily.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void composite_terms_keyword() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/composite_terms_keyword.ppl")); + String ppl = sanitize(loadExpectedQuery("composite_terms_keyword.ppl")); timing(summary, "composite_terms_keyword", ppl); + String expected = loadExpectedPlan("composite_terms_keyword.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void composite_terms() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/composite_terms.ppl")); + String ppl = sanitize(loadExpectedQuery("composite_terms.ppl")); timing(summary, "composite_terms", ppl); + String expected = loadExpectedPlan("composite_terms.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void date_histogram_hourly_agg() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/date_histogram_hourly_agg.ppl")); + String ppl = sanitize(loadExpectedQuery("date_histogram_hourly_agg.ppl")); timing(summary, "date_histogram_hourly_agg", ppl); + String expected = loadExpectedPlan("date_histogram_hourly_agg.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void date_histogram_minute_agg() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/date_histogram_minute_agg.ppl")); + String ppl = sanitize(loadExpectedQuery("date_histogram_minute_agg.ppl")); timing(summary, "date_histogram_minute_agg", ppl); + String expected = loadExpectedPlan("date_histogram_minute_agg.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void test_default() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/default.ppl")); + String ppl = sanitize(loadExpectedQuery("default.ppl")); timing(summary, "default", ppl); + String expected = loadExpectedPlan("default.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void desc_sort_timestamp() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/desc_sort_timestamp.ppl")); + String ppl = sanitize(loadExpectedQuery("desc_sort_timestamp.ppl")); timing(summary, "desc_sort_timestamp", ppl); + String expected = loadExpectedPlan("desc_sort_timestamp.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void desc_sort_timestamp_can_match_shortcut() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/desc_sort_timestamp_can_match_shortcut.ppl")); + String ppl = sanitize(loadExpectedQuery("desc_sort_timestamp_can_match_shortcut.ppl")); timing(summary, "desc_sort_timestamp_can_match_shortcut", ppl); + String expected = loadExpectedPlan("desc_sort_timestamp_can_match_shortcut.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void desc_sort_timestamp_no_can_match_shortcut() throws IOException { - String ppl = - sanitize(loadFromFile("big5/queries/desc_sort_timestamp_no_can_match_shortcut.ppl")); + String ppl = sanitize(loadExpectedQuery("desc_sort_timestamp_no_can_match_shortcut.ppl")); timing(summary, "desc_sort_timestamp_no_can_match_shortcut", ppl); + String expected = loadExpectedPlan("desc_sort_timestamp_no_can_match_shortcut.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void desc_sort_with_after_timestamp() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/desc_sort_with_after_timestamp.ppl")); + String ppl = sanitize(loadExpectedQuery("desc_sort_with_after_timestamp.ppl")); timing(summary, "desc_sort_with_after_timestamp", ppl); + String expected = loadExpectedPlan("desc_sort_with_after_timestamp.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void keyword_in_range() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/keyword_in_range.ppl")); + String ppl = sanitize(loadExpectedQuery("keyword_in_range.ppl")); timing(summary, "keyword_in_range", ppl); + String expected = loadExpectedPlan("keyword_in_range.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void keyword_terms() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/keyword_terms.ppl")); + String ppl = sanitize(loadExpectedQuery("keyword_terms.ppl")); timing(summary, "keyword_terms", ppl); + String expected = loadExpectedPlan("keyword_terms.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void keyword_terms_low_cardinality() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/keyword_terms_low_cardinality.ppl")); + String ppl = sanitize(loadExpectedQuery("keyword_terms_low_cardinality.ppl")); timing(summary, "keyword_terms_low_cardinality", ppl); + String expected = loadExpectedPlan("keyword_terms_low_cardinality.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void multi_terms_keyword() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/multi_terms_keyword.ppl")); + String ppl = sanitize(loadExpectedQuery("multi_terms_keyword.ppl")); timing(summary, "multi_terms_keyword", ppl); + String expected = loadExpectedPlan("multi_terms_keyword.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void query_string_on_message() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/query_string_on_message.ppl")); + String ppl = sanitize(loadExpectedQuery("query_string_on_message.ppl")); timing(summary, "query_string_on_message", ppl); + String expected = loadExpectedPlan("query_string_on_message.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void query_string_on_message_filtered() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/query_string_on_message_filtered.ppl")); + String ppl = sanitize(loadExpectedQuery("query_string_on_message_filtered.ppl")); timing(summary, "query_string_on_message_filtered", ppl); + String expected = loadExpectedPlan("query_string_on_message_filtered.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void query_string_on_message_filtered_sorted_num() throws IOException { - String ppl = - sanitize(loadFromFile("big5/queries/query_string_on_message_filtered_sorted_num.ppl")); + String ppl = sanitize(loadExpectedQuery("query_string_on_message_filtered_sorted_num.ppl")); timing(summary, "query_string_on_message_filtered_sorted_num", ppl); + String expected = loadExpectedPlan("query_string_on_message_filtered_sorted_num.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void range() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/range.ppl")); + String ppl = sanitize(loadExpectedQuery("range.ppl")); timing(summary, "range", ppl); + String expected = loadExpectedPlan("range.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void range_auto_date_histo() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/range_auto_date_histo.ppl")); + String ppl = sanitize(loadExpectedQuery("range_auto_date_histo.ppl")); timing(summary, "range_auto_date_histo", ppl); + String expected = loadExpectedPlan("range_auto_date_histo.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void range_auto_date_histo_with_metrics() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/range_auto_date_histo_with_metrics.ppl")); + String ppl = sanitize(loadExpectedQuery("range_auto_date_histo_with_metrics.ppl")); timing(summary, "range_auto_date_histo_with_metrics", ppl); + String expected = loadExpectedPlan("range_auto_date_histo_with_metrics.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void range_numeric() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/range_numeric.ppl")); + String ppl = sanitize(loadExpectedQuery("range_numeric.ppl")); timing(summary, "range_numeric", ppl); + String expected = loadExpectedPlan("range_numeric.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void range_field_conjunction_big_range_big_term_query() throws IOException { String ppl = - sanitize(loadFromFile("big5/queries/range_field_conjunction_big_range_big_term_query.ppl")); + sanitize(loadExpectedQuery("range_field_conjunction_big_range_big_term_query.ppl")); timing(summary, "range_field_conjunction_big_range_big_term_query", ppl); + String expected = loadExpectedPlan("range_field_conjunction_big_range_big_term_query.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void range_field_conjunction_small_range_big_term_query() throws IOException { String ppl = - sanitize( - loadFromFile("big5/queries/range_field_conjunction_small_range_big_term_query.ppl")); + sanitize(loadExpectedQuery("range_field_conjunction_small_range_big_term_query.ppl")); timing(summary, "range_field_conjunction_small_range_big_term_query", ppl); + String expected = loadExpectedPlan("range_field_conjunction_small_range_big_term_query.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void range_field_conjunction_small_range_small_term_query() throws IOException { String ppl = - sanitize( - loadFromFile("big5/queries/range_field_conjunction_small_range_small_term_query.ppl")); + sanitize(loadExpectedQuery("range_field_conjunction_small_range_small_term_query.ppl")); timing(summary, "range_field_conjunction_small_range_small_term_query", ppl); + String expected = loadExpectedPlan("range_field_conjunction_small_range_small_term_query.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void range_field_disjunction_big_range_small_term_query() throws IOException { String ppl = - sanitize( - loadFromFile("big5/queries/range_field_disjunction_big_range_small_term_query.ppl")); + sanitize(loadExpectedQuery("range_field_disjunction_big_range_small_term_query.ppl")); timing(summary, "range_field_disjunction_big_range_small_term_query", ppl); + String expected = loadExpectedPlan("range_field_disjunction_big_range_small_term_query.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void range_with_asc_sort() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/range_with_asc_sort.ppl")); + String ppl = sanitize(loadExpectedQuery("range_with_asc_sort.ppl")); timing(summary, "range_with_asc_sort", ppl); + String expected = loadExpectedPlan("range_with_asc_sort.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void range_with_desc_sort() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/range_with_desc_sort.ppl")); + String ppl = sanitize(loadExpectedQuery("range_with_desc_sort.ppl")); timing(summary, "range_with_desc_sort", ppl); + String expected = loadExpectedPlan("range_with_desc_sort.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void scroll() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/scroll.ppl")); + String ppl = sanitize(loadExpectedQuery("scroll.ppl")); timing(summary, "scroll", ppl); + String expected = loadExpectedPlan("scroll.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void sort_keyword_can_match_shortcut() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/sort_keyword_can_match_shortcut.ppl")); + String ppl = sanitize(loadExpectedQuery("sort_keyword_can_match_shortcut.ppl")); timing(summary, "sort_keyword_can_match_shortcut", ppl); + String expected = loadExpectedPlan("sort_keyword_can_match_shortcut.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void sort_keyword_no_can_match_shortcut() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/sort_keyword_no_can_match_shortcut.ppl")); + String ppl = sanitize(loadExpectedQuery("sort_keyword_no_can_match_shortcut.ppl")); timing(summary, "sort_keyword_no_can_match_shortcut", ppl); + String expected = loadExpectedPlan("sort_keyword_no_can_match_shortcut.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void sort_numeric_asc() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/sort_numeric_asc.ppl")); + String ppl = sanitize(loadExpectedQuery("sort_numeric_asc.ppl")); timing(summary, "sort_numeric_asc", ppl); + String expected = loadExpectedPlan("sort_numeric_asc.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void sort_numeric_asc_with_match() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/sort_numeric_asc_with_match.ppl")); + String ppl = sanitize(loadExpectedQuery("sort_numeric_asc_with_match.ppl")); timing(summary, "sort_numeric_asc_with_match", ppl); + String expected = loadExpectedPlan("sort_numeric_asc_with_match.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void sort_numeric_desc() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/sort_numeric_desc.ppl")); + String ppl = sanitize(loadExpectedQuery("sort_numeric_desc.ppl")); timing(summary, "sort_numeric_desc", ppl); + String expected = loadExpectedPlan("sort_numeric_desc.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void sort_numeric_desc_with_match() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/sort_numeric_desc_with_match.ppl")); + String ppl = sanitize(loadExpectedQuery("sort_numeric_desc_with_match.ppl")); timing(summary, "sort_numeric_desc_with_match", ppl); + String expected = loadExpectedPlan("sort_numeric_desc_with_match.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void term() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/term.ppl")); + String ppl = sanitize(loadExpectedQuery("term.ppl")); timing(summary, "term", ppl); + String expected = loadExpectedPlan("term.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void terms_significant_1() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/terms_significant_1.ppl")); + String ppl = sanitize(loadExpectedQuery("terms_significant_1.ppl")); timing(summary, "terms_significant_1", ppl); + String expected = loadExpectedPlan("terms_significant_1.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void terms_significant_2() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/terms_significant_2.ppl")); + String ppl = sanitize(loadExpectedQuery("terms_significant_2.ppl")); timing(summary, "terms_significant_2", ppl); + String expected = loadExpectedPlan("terms_significant_2.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); + } + + @Test + public void range_agg_1() throws IOException { + String ppl = sanitize(loadExpectedQuery("range_agg_1.ppl")); + timing(summary, "range_agg_1", ppl); + String expected = loadExpectedPlan("range_agg_1.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); + } + + @Test + public void range_agg_2() throws IOException { + String ppl = sanitize(loadExpectedQuery("range_agg_2.ppl")); + timing(summary, "range_agg_2", ppl); + String expected = loadExpectedPlan("range_agg_2.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); + } + + @Test + public void cardinality_agg_high() throws IOException { + String ppl = sanitize(loadExpectedQuery("cardinality_agg_high.ppl")); + timing(summary, "cardinality_agg_high", ppl); + String expected = loadExpectedPlan("cardinality_agg_high.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); + } + + @Test + public void cardinality_agg_high_2() throws IOException { + String ppl = sanitize(loadExpectedQuery("cardinality_agg_high_2.ppl")); + timing(summary, "cardinality_agg_high_2", ppl); + String expected = loadExpectedPlan("cardinality_agg_high_2.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); + } + + @Test + public void cardinality_agg_low() throws IOException { + String ppl = sanitize(loadExpectedQuery("cardinality_agg_low.ppl")); + timing(summary, "cardinality_agg_low", ppl); + String expected = loadExpectedPlan("cardinality_agg_low.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); + } + + protected String loadExpectedQuery(String fileName) throws IOException { + if (isCalciteEnabled()) { + try { + return loadFromFile("big5/queries/optimized/" + fileName); + } catch (Exception e) { + } + } + return loadFromFile("big5/queries/" + fileName); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/clickbench/PPLClickBenchIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/clickbench/PPLClickBenchIT.java index a2ce3d7841f..a4ea7dbe183 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/clickbench/PPLClickBenchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/clickbench/PPLClickBenchIT.java @@ -5,11 +5,19 @@ package org.opensearch.sql.calcite.clickbench; +import static org.opensearch.sql.util.MatcherUtils.assertJsonEquals; +import static org.opensearch.sql.util.MatcherUtils.assertYamlEqualsIgnoreId; + import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; + import org.junit.AfterClass; +import org.junit.Assert; import org.junit.FixMethodOrder; import org.junit.Test; import org.junit.runners.MethodSorters; @@ -20,6 +28,10 @@ @FixMethodOrder(MethodSorters.JVM) public class PPLClickBenchIT extends PPLIntegTestCase { private static final MapBuilder summary = MapBuilder.newMapBuilder(); + private static final MapBuilder response_200 = MapBuilder.newMapBuilder(); + private static final List response_200_failing = new java.util.ArrayList(); + private static final List non_200 = new java.util.ArrayList(); + private static final List passing = new java.util.ArrayList(); @Override public void init() throws Exception { @@ -37,27 +49,33 @@ public static void reset() throws IOException { } System.out.println("Summary:"); map.entrySet().stream() - .sorted(Map.Entry.comparingByKey()) - .forEach( - entry -> - System.out.printf(Locale.ENGLISH, "%s: %d ms%n", entry.getKey(), entry.getValue())); + .sorted(Map.Entry.comparingByKey()) + .forEach( + entry -> + System.out.printf(Locale.ENGLISH, "%s: %d ms%n", entry.getKey(), entry.getValue())); System.out.printf( - Locale.ENGLISH, - "Total %d queries succeed. Average duration: %d ms%n", - map.size(), - total / map.size()); + Locale.ENGLISH, + "Total %d queries succeed. Average duration: %d ms%n", + map.size(), + total / map.size()); System.out.println(); + + // Display detailed query results + if (!passing.isEmpty() || !response_200_failing.isEmpty() || !non_200.isEmpty()) { + System.out.printf( + Locale.ENGLISH, + "Total: %d | Passed: %d | Failed (200): %d | Failed (non-200): %d%n", + passing.size() + response_200_failing.size() + non_200.size(), + passing.size(), + response_200_failing.size(), + non_200.size()); + System.out.println(); + } } /** Ignore queries that are not supported by Calcite. */ protected Set ignored() { - if (GCedMemoryUsage.initialized()) { - return Set.of(29); - } else { - // Ignore q30 when use RuntimeMemoryUsage, - // because of too much script push down, which will cause ResourceMonitor restriction. - return Set.of(29, 30); - } + return Set.of(41); // query currently fails on main } @Test @@ -66,8 +84,73 @@ public void test() throws IOException { if (ignored().contains(i)) { continue; } + logger.info("Running Query{}", i); String ppl = sanitize(loadFromFile("clickbench/queries/q" + i + ".ppl")); + // V2 gets unstable scripts, ignore them when comparing plan + if (isCalciteEnabled()) { + String expected = loadExpectedPlan("clickbench/q" + i + ".yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); + } timing(summary, "q" + i, ppl); } } + + /** Queries that are returning 200s and response is correct and not empty */ + protected Set df_ignored() { + return Set.of(4, 9, 10, 11, 12, 14, 19, 20, 24, 25, 26, 27, 28, 29, 30, 39, 40, 41, 42, 43); + } + + @Test + public void testDataFusion() throws IOException { + // flip this to run everything and get the full current list of p/f/f200. + // when false will fail on first f200 occurence and show assert diff. + boolean runAllQueries = true; + Set ignore = df_ignored(); + for (int i = 1; i <= 43; i++) { + if (ignored().contains(i)) { + continue; + } + String ppl = sanitize(loadFromFile("clickbench/queries/q" + i + ".ppl")); + System.out.println("RUNNING QUERY NUMBER: " + i + " Query: " + ppl); + + // TODO: Add plan comparisons +// if (isCalciteEnabled()) { +// String expected = loadExpectedPlan("clickbench/q" + i + ".yaml"); +// assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); +// } + // runs the query and buckets into failing (non200), passing (200 and response matches), failing_200 + String actual = runQuery(summary, response_200, i, ppl); + String expected = sanitize(loadFromFile("clickbench/queries/expected/expected-q" + i + ".json")); + + if (response_200.get(i)) { + // 200 returned and we expect it to pass + try { + assertJsonEquals(String.format("query number %d", i), expected, actual); + passing.add(i); + } catch (AssertionError e) { + // comment this out to get a full list of current pass/failed + if (!ignore.contains(i) && runAllQueries == false) { + throw e; + } + response_200_failing.add(i); + // 200 but we haven't marked supported yet, mark it in a separate list + } + } else { + non_200.add(i); + } + } + // display results + System.out.println("PASSING: " + passing); + System.out.println("FAILING WITH 200: " + response_200_failing); + System.out.println("FAILING: " + non_200); + + List failed = new ArrayList<>(); + failed.addAll(response_200_failing); + failed.addAll(non_200); + List supportedButNotPassing = failed.stream() + .filter(i -> !ignore.contains(i)) + .sorted() + .toList(); + assertEquals("Expected all supported queries to be marked passing", Collections.emptyList(), supportedButNotPassing); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java new file mode 100644 index 00000000000..0b15a36d6d8 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java @@ -0,0 +1,158 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; +import static org.opensearch.sql.util.MatcherUtils.verifySchemaInOrder; + +import java.io.IOException; +import java.util.List; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** + * Integration tests for aggregation functions (MIN, MAX, FIRST, LAST, TAKE) with alias fields. + * Tests the fix for issue #4595. + */ +public class CalciteAliasFieldAggregationIT extends PPLIntegTestCase { + + private static final String TEST_INDEX_ALIAS = "test_alias_bug"; + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + createTestIndexWithAliasFields(); + } + + /** + * Create test index with alias fields mapping and insert sample data. This mirrors the + * reproduction steps from issue #4595. + */ + private void createTestIndexWithAliasFields() throws IOException { + // Delete the index if it exists (for test isolation) + try { + Request deleteIndex = new Request("DELETE", "/" + TEST_INDEX_ALIAS); + client().performRequest(deleteIndex); + } catch (ResponseException e) { + // Index doesn't exist, which is fine + } + + // Create index with alias fields + Request createIndex = new Request("PUT", "/" + TEST_INDEX_ALIAS); + createIndex.setJsonEntity( + "{\n" + + " \"mappings\": {\n" + + " \"properties\": {\n" + + " \"created_at\": {\"type\": \"date\"},\n" + + " \"@timestamp\": {\"type\": \"alias\", \"path\": \"created_at\"},\n" + + " \"value\": {\"type\": \"integer\"},\n" + + " \"value_alias\": {\"type\": \"alias\", \"path\": \"value\"}\n" + + " }\n" + + " }\n" + + "}"); + client().performRequest(createIndex); + + // Insert test documents + Request bulkRequest = new Request("POST", "/" + TEST_INDEX_ALIAS + "/_bulk?refresh=true"); + bulkRequest.setJsonEntity( + "{\"index\":{}}\n" + + "{\"created_at\": \"2024-01-01T10:00:00Z\", \"value\": 100}\n" + + "{\"index\":{}}\n" + + "{\"created_at\": \"2024-01-02T10:00:00Z\", \"value\": 200}\n" + + "{\"index\":{}}\n" + + "{\"created_at\": \"2024-01-03T10:00:00Z\", \"value\": 300}\n"); + client().performRequest(bulkRequest); + } + + @Test + public void testMinWithDateAliasField() throws IOException { + JSONObject actual = + executeQuery(String.format("source=%s | stats MIN(@timestamp)", TEST_INDEX_ALIAS)); + verifySchema(actual, schema("MIN(@timestamp)", "timestamp")); + verifyDataRows(actual, rows("2024-01-01 10:00:00")); + } + + @Test + public void testMaxWithDateAliasField() throws IOException { + JSONObject actual = + executeQuery(String.format("source=%s | stats MAX(@timestamp)", TEST_INDEX_ALIAS)); + verifySchema(actual, schema("MAX(@timestamp)", "timestamp")); + verifyDataRows(actual, rows("2024-01-03 10:00:00")); + } + + @Test + public void testMinMaxWithNumericAliasField() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | stats MIN(value_alias), MAX(value_alias)", TEST_INDEX_ALIAS)); + verifySchemaInOrder( + actual, schema("MIN(value_alias)", "int"), schema("MAX(value_alias)", "int")); + verifyDataRows(actual, rows(100, 300)); + } + + @Test + public void testFirstWithAliasField() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | sort @timestamp | stats FIRST(@timestamp)", TEST_INDEX_ALIAS)); + verifySchema(actual, schema("FIRST(@timestamp)", "timestamp")); + verifyDataRows(actual, rows("2024-01-01 10:00:00")); + } + + @Test + public void testLastWithAliasField() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | sort @timestamp | stats LAST(@timestamp)", TEST_INDEX_ALIAS)); + verifySchema(actual, schema("LAST(@timestamp)", "timestamp")); + verifyDataRows(actual, rows("2024-01-03 10:00:00")); + } + + @Test + public void testTakeWithAliasField() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | sort @timestamp | stats TAKE(@timestamp, 2)", TEST_INDEX_ALIAS)); + verifySchema(actual, schema("TAKE(@timestamp, 2)", "array")); + verifyDataRows(actual, rows(List.of("2024-01-01T10:00:00.000Z", "2024-01-02T10:00:00.000Z"))); + } + + @Test + public void testAggregationsWithOriginalFieldsStillWork() throws IOException { + JSONObject actual = + executeQuery( + String.format("source=%s | stats MIN(created_at), MAX(value)", TEST_INDEX_ALIAS)); + verifySchemaInOrder( + actual, schema("MIN(created_at)", "timestamp"), schema("MAX(value)", "int")); + verifyDataRows(actual, rows("2024-01-01 10:00:00", 300)); + } + + @Test + public void testUnaffectedAggregationsWithAliasFields() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | stats SUM(value_alias), AVG(value_alias), COUNT(value_alias)", + TEST_INDEX_ALIAS)); + verifySchemaInOrder( + actual, + schema("SUM(value_alias)", "bigint"), + schema("AVG(value_alias)", "double"), + schema("COUNT(value_alias)", "bigint")); + verifyDataRows(actual, rows(600, 200.0, 3)); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteArrayFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteArrayFunctionIT.java index 7aa448bf45a..458158c45d4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteArrayFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteArrayFunctionIT.java @@ -67,7 +67,7 @@ public void testArrayWithMix() { verifyErrorMessageContains( e, "Cannot resolve function: ARRAY, arguments: [INTEGER,BOOLEAN], caused by: fail to create" - + " array with fixed type: inferred array element type"); + + " array with fixed type"); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java index 2aa4455fb72..ef8abc3cba0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.calcite.remote; +import static org.junit.Assert.assertTrue; import static org.opensearch.sql.legacy.TestsConstants.*; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; @@ -27,6 +28,7 @@ public void init() throws Exception { loadIndex(Index.BANK); loadIndex(Index.EVENTS_NULL); loadIndex(Index.TIME_TEST_DATA); + loadIndex(Index.TELEMETRY); } @Test @@ -505,6 +507,145 @@ public void testBinWithNonExistentField() { errorMessage.contains("non_existent_field") || errorMessage.contains("not found")); } + @Test + public void testBinWithMinspanOnNonNumericField() { + // Test that bin command with minspan throws clear error for non-numeric field + ResponseException exception = + assertThrows( + ResponseException.class, + () -> { + executeQuery( + String.format( + "source=%s | bin firstname minspan=10 | head 1", TEST_INDEX_ACCOUNT)); + }); + + // Get the full error message + String errorMessage = exception.getMessage(); + + // Verify the error message is clear and specific + String expectedMessage = + "Cannot apply binning: field 'firstname' is non-numeric and not time-related, expected" + + " numeric or time-related type"; + assertTrue( + "Error message should contain: '" + expectedMessage + "'", + errorMessage.contains(expectedMessage)); + } + + @Test + public void testBinWithSpanOnNonNumericField() { + // Test that bin command with span throws clear error for non-numeric field + ResponseException exception = + assertThrows( + ResponseException.class, + () -> { + executeQuery( + String.format("source=%s | bin lastname span=5 | head 1", TEST_INDEX_ACCOUNT)); + }); + + // Get the full error message + String errorMessage = exception.getMessage(); + + // Verify the error message is clear and specific + String expectedMessage = + "Cannot apply binning: field 'lastname' is non-numeric and not time-related, expected" + + " numeric or time-related type"; + assertTrue( + "Error message should contain: '" + expectedMessage + "'", + errorMessage.contains(expectedMessage)); + } + + @Test + public void testBinWithBinsOnNonNumericField() { + // Test that bin command with bins throws clear error for non-numeric field + ResponseException exception = + assertThrows( + ResponseException.class, + () -> { + executeQuery( + String.format("source=%s | bin state bins=10 | head 1", TEST_INDEX_ACCOUNT)); + }); + + // Get the full error message + String errorMessage = exception.getMessage(); + + // Verify the error message is clear and specific + String expectedMessage = + "Cannot apply binning: field 'state' is non-numeric and not time-related, expected numeric" + + " or time-related type"; + assertTrue( + "Error message should contain: '" + expectedMessage + "'", + errorMessage.contains(expectedMessage)); + } + + @Test + public void testBinWithStartEndOnNonNumericField() { + // Test that bin command with start/end throws clear error for non-numeric field + ResponseException exception = + assertThrows( + ResponseException.class, + () -> { + executeQuery( + String.format( + "source=%s | bin city start=0 end=100 | head 1", TEST_INDEX_ACCOUNT)); + }); + + // Get the full error message + String errorMessage = exception.getMessage(); + + // Verify the error message is clear and specific + String expectedMessage = + "Cannot apply binning: field 'city' is non-numeric and not time-related, expected numeric" + + " or time-related type"; + assertTrue( + "Error message should contain: '" + expectedMessage + "'", + errorMessage.contains(expectedMessage)); + } + + @Test + public void testBinDefaultOnNonNumericField() { + // Test that default bin (no parameters) throws clear error for non-numeric field + ResponseException exception = + assertThrows( + ResponseException.class, + () -> { + executeQuery(String.format("source=%s | bin email | head 1", TEST_INDEX_ACCOUNT)); + }); + + // Get the full error message + String errorMessage = exception.getMessage(); + + // Verify the error message is clear and specific + String expectedMessage = + "Cannot apply binning: field 'email' is non-numeric and not time-related, expected numeric" + + " or time-related type"; + assertTrue( + "Error message should contain: '" + expectedMessage + "'", + errorMessage.contains(expectedMessage)); + } + + @Test + public void testBinLogSpanOnNonNumericField() { + // Test that bin command with log span throws clear error for non-numeric field + ResponseException exception = + assertThrows( + ResponseException.class, + () -> { + executeQuery( + String.format("source=%s | bin gender span=log10 | head 1", TEST_INDEX_ACCOUNT)); + }); + + // Get the full error message + String errorMessage = exception.getMessage(); + + // Verify the error message is clear and specific + String expectedMessage = + "Cannot apply binning: field 'gender' is non-numeric and not time-related, expected numeric" + + " or time-related type"; + assertTrue( + "Error message should contain: '" + expectedMessage + "'", + errorMessage.contains(expectedMessage)); + } + @Test public void testBinSpanWithStartEndNeverShrinkRange() throws IOException { JSONObject result = @@ -867,9 +1008,8 @@ public void testStatsWithBinsOnTimeField_Count() throws IOException { JSONObject result = executeQuery("source=events_null | bin @timestamp bins=3 | stats count() by @timestamp"); - // TODO: @timestamp should keep date as its type, to be addressed by this issue: - // https://github.com/opensearch-project/sql/issues/4317 - verifySchema(result, schema("count()", null, "bigint"), schema("@timestamp", null, "string")); + verifySchema( + result, schema("count()", null, "bigint"), schema("@timestamp", null, "timestamp")); // auto_date_histogram will choose span=5m for bins=3 verifyDataRows(result, rows(5, "2024-07-01 00:00:00"), rows(1, "2024-07-01 00:05:00")); @@ -907,10 +1047,8 @@ public void testStatsWithBinsOnTimeField_Avg() throws IOException { JSONObject result = executeQuery( "source=events_null | bin @timestamp bins=3 | stats avg(cpu_usage) by @timestamp"); - // TODO: @timestamp should keep date as its type, to be addressed by this issue: - // https://github.com/opensearch-project/sql/issues/4317 verifySchema( - result, schema("avg(cpu_usage)", null, "double"), schema("@timestamp", null, "string")); + result, schema("avg(cpu_usage)", null, "double"), schema("@timestamp", null, "timestamp")); // auto_date_histogram will choose span=5m for bins=3 verifyDataRows(result, rows(44.62, "2024-07-01 00:00:00"), rows(50.0, "2024-07-01 00:05:00")); @@ -941,4 +1079,131 @@ public void testStatsWithBinsOnTimeField_Avg() throws IOException { rows(41.8, "2024-07-01 00:04:00"), rows(50.0, "2024-07-01 00:05:00")); } + + @Test + public void testStatsWithBinsOnTimeAndTermField_Count() throws IOException { + // TODO: Remove this after addressing https://github.com/opensearch-project/sql/issues/4317 + enabledOnlyWhenPushdownIsEnabled(); + + JSONObject result = + executeQuery( + "source=events_null | bin @timestamp bins=3 | stats bucket_nullable=false count() by" + + " region, @timestamp"); + verifySchema( + result, + schema("count()", null, "bigint"), + schema("region", null, "string"), + schema("@timestamp", null, "timestamp")); + // auto_date_histogram will choose span=5m for bins=3 + verifyDataRows( + result, + rows(1, "eu-west", "2024-07-01 00:03:00"), + rows(2, "us-east", "2024-07-01 00:00:00"), + rows(1, "us-east", "2024-07-01 00:05:00"), + rows(2, "us-west", "2024-07-01 00:01:00")); + } + + @Test + public void testStatsWithBinsOnTimeAndTermField_Avg() throws IOException { + // TODO: Remove this after addressing https://github.com/opensearch-project/sql/issues/4317 + enabledOnlyWhenPushdownIsEnabled(); + + JSONObject result = + executeQuery( + "source=events_null | bin @timestamp bins=3 | stats bucket_nullable=false " + + " avg(cpu_usage) by region, @timestamp"); + verifySchema( + result, + schema("avg(cpu_usage)", null, "double"), + schema("region", null, "string"), + schema("@timestamp", null, "timestamp")); + // auto_date_histogram will choose span=5m for bins=3 + verifyDataRows( + result, + rows(42.1, "eu-west", "2024-07-01 00:03:00"), + rows(50.25, "us-east", "2024-07-01 00:00:00"), + rows(50, "us-east", "2024-07-01 00:05:00"), + rows(40.25, "us-west", "2024-07-01 00:01:00")); + } + + @Test + public void testBinWithNestedFieldWithoutExplicitProjection() throws IOException { + // Test bin command on nested field without explicit fields projection + // This reproduces the bug from https://github.com/opensearch-project/sql/issues/4482 + // The telemetry index has: resource.attributes.telemetry.sdk.version (values: 10, 11, 12, 13, + // 14) + JSONObject result = + executeQuery( + String.format( + "source=%s | bin `resource.attributes.telemetry.sdk.version` span=2 | sort" + + " `resource.attributes.telemetry.sdk.version`", + TEST_INDEX_TELEMETRY)); + + // When binning a nested field, all sibling fields in the struct are also returned + verifySchema( + result, + schema("resource.attributes.telemetry.sdk.enabled", null, "boolean"), + schema("resource.attributes.telemetry.sdk.language", null, "string"), + schema("resource.attributes.telemetry.sdk.name", null, "string"), + schema("severityNumber", null, "int"), + schema("resource.attributes.telemetry.sdk.version", null, "string")); + + // With span=2 on values [10, 11, 12, 13, 14], we expect binned ranges: + // 10 -> 10-12, 11 -> 10-12, 12 -> 12-14, 13 -> 12-14, 14 -> 14-16 + // The binned field is the last column + verifyDataRows( + result, + rows(true, "java", "opentelemetry", 9, "10-12"), + rows(false, "python", "opentelemetry", 12, "10-12"), + rows(true, "javascript", "opentelemetry", 9, "12-14"), + rows(false, "go", "opentelemetry", 16, "12-14"), + rows(true, "rust", "opentelemetry", 12, "14-16")); + } + + @Test + public void testBinWithNestedFieldWithExplicitProjection() throws IOException { + // Test bin command on nested field WITH explicit fields projection (workaround) + // This is the workaround mentioned in https://github.com/opensearch-project/sql/issues/4482 + JSONObject result = + executeQuery( + String.format( + "source=%s | bin `resource.attributes.telemetry.sdk.version` span=2 | fields" + + " `resource.attributes.telemetry.sdk.version` | sort" + + " `resource.attributes.telemetry.sdk.version`", + TEST_INDEX_TELEMETRY)); + verifySchema(result, schema("resource.attributes.telemetry.sdk.version", null, "string")); + + // With span=2 on values [10, 11, 12, 13, 14], we expect binned ranges + verifyDataRows( + result, rows("10-12"), rows("10-12"), rows("12-14"), rows("12-14"), rows("14-16")); + } + + @Test + public void testBinWithEvalCreatedDottedFieldName() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | eval `resource.temp` = 1 | bin" + + " `resource.attributes.telemetry.sdk.version` span=2 | sort" + + " `resource.attributes.telemetry.sdk.version`", + TEST_INDEX_TELEMETRY)); + + verifySchema( + result, + schema("resource.attributes.telemetry.sdk.enabled", null, "boolean"), + schema("resource.attributes.telemetry.sdk.language", null, "string"), + schema("resource.attributes.telemetry.sdk.name", null, "string"), + schema("resource.temp", null, "int"), + schema("severityNumber", null, "int"), + schema("resource.attributes.telemetry.sdk.version", null, "string")); + + // Data column order: enabled, language, name, severityNumber, resource.temp, version + verifyDataRows( + result, + rows(true, "java", "opentelemetry", 9, 1, "10-12"), + rows(false, "python", "opentelemetry", 12, 1, "10-12"), + rows(true, "javascript", "opentelemetry", 9, 1, "12-14"), + rows(false, "go", "opentelemetry", 16, 1, "12-14"), + rows(true, "rust", "opentelemetry", 12, 1, "14-16")); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDateTimeFunctionIT.java index 8eee5c01f7c..ef0c0599b57 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDateTimeFunctionIT.java @@ -132,22 +132,13 @@ public void testStrftimeWithExpressions() throws IOException { @Test public void testStrftimeStringHandling() throws IOException { - try { - executeQuery( - String.format( - "source=%s | eval result = strftime('1521467703', '%s') | fields result | head 1", - TEST_INDEX_DATE, "%Y-%m-%d")); - fail("String literals should not be accepted by strftime"); - } catch (Exception e) { - // Expected - string literals are not supported - // The error occurs because Calcite tries to convert the string to a timestamp - // which doesn't match the expected timestamp format - assertTrue( - "Error should indicate format issue or type problem", - e.getMessage().contains("unsupported format") - || e.getMessage().contains("timestamp") - || e.getMessage().contains("500")); - } + // Test 1: Support string literal + JSONObject result0 = + executeQuery( + String.format( + "source=%s | eval result = strftime('1521467703', '%s') | fields result | head 1", + TEST_INDEX_DATE, "%Y-%m-%d")); + verifyDataRows(result0, rows("2018-03-19")); // Test 2: The correct approach - use numeric literals directly JSONObject result1 = diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 279cfd94b3c..6da047e0c20 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -10,14 +10,18 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_LOGS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_SIMPLE; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STRINGS; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_TIME_DATA; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOGS; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WORKER; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WORK_INFORMATION; import static org.opensearch.sql.util.MatcherUtils.assertJsonEqualsIgnoreId; -import static org.opensearch.sql.util.MatcherUtils.assertYamlEqualsJsonIgnoreId; +import static org.opensearch.sql.util.MatcherUtils.assertYamlEqualsIgnoreId; import java.io.IOException; import java.util.Locale; import org.junit.Ignore; import org.junit.Test; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.ppl.ExplainIT; public class CalciteExplainIT extends ExplainIT { @@ -25,11 +29,16 @@ public class CalciteExplainIT extends ExplainIT { public void init() throws Exception { super.init(); enableCalcite(); + setQueryBucketSize(1000); loadIndex(Index.BANK_WITH_STRING_VALUES); loadIndex(Index.NESTED_SIMPLE); loadIndex(Index.TIME_TEST_DATA); + loadIndex(Index.TIME_TEST_DATA2); loadIndex(Index.EVENTS); loadIndex(Index.LOGS); + loadIndex(Index.WORKER); + loadIndex(Index.WORK_INFORMATION); + loadIndex(Index.WEBLOG); } @Override @@ -41,9 +50,9 @@ public void testExplainModeUnsupportedInV2() throws IOException {} public void supportSearchSargPushDown_singleRange() throws IOException { String query = "source=opensearch-sql_test_index_account | where age >= 1.0 and age < 10 | fields age"; - var result = explainQueryToString(query); + var result = explainQueryYaml(query); String expected = loadExpectedPlan("explain_sarg_filter_push_single_range.yaml"); - assertYamlEqualsJsonIgnoreId(expected, result); + assertYamlEqualsIgnoreId(expected, result); } // Only for Calcite @@ -64,9 +73,9 @@ public void supportSearchSargPushDown_timeRange() throws IOException { "source=opensearch-sql_test_index_bank" + "| where birthdate >= '2016-12-08 00:00:00.000000000' " + "and birthdate < '2018-11-09 00:00:00.000000000'"; - var result = explainQueryToString(query); - String expected = loadExpectedPlan("explain_sarg_filter_push_time_range.json"); - assertJsonEqualsIgnoreId(expected, result); + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_sarg_filter_push_time_range.yaml"); + assertYamlEqualsIgnoreId(expected, result); } // Only for Calcite @@ -97,9 +106,133 @@ public void testJoinWithFieldList() throws IOException { String query = "source=opensearch-sql_test_index_bank | join type=outer account_number" + " opensearch-sql_test_index_bank"; - var result = explainQueryToString(query); - String expected = loadExpectedPlan("explain_join_with_fields.json"); - assertJsonEqualsIgnoreId(expected, result); + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_join_with_fields.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + + @Test + public void testExplainExistsUncorrelatedSubquery() throws IOException { + String expected = loadExpectedPlan("explain_exists_uncorrelated_subquery.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source = %s" + + "| where exists [" + + " source = %s | where name = 'Tom'" + + " ]" + + "| sort - salary" + + "| fields id, name, salary", + TEST_INDEX_WORKER, TEST_INDEX_WORK_INFORMATION))); + } + + @Test + public void testExplainExistsCorrelatedSubquery() throws IOException { + String expected = loadExpectedPlan("explain_exists_correlated_subquery.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source = %s" + + "| where exists [" + + " source = %s | where id = uid and name = 'Tom'" + + " ]" + + "| sort - salary" + + "| fields id, name, salary", + TEST_INDEX_WORKER, TEST_INDEX_WORK_INFORMATION))); + } + + @Test + public void testExplainInUncorrelatedSubquery() throws IOException { + String expected = loadExpectedPlan("explain_in_uncorrelated_subquery.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source = %s" + + "| where id in [" + + " source = %s | fields uid" + + " ]" + + "| sort - salary" + + "| fields id, name, salary", + TEST_INDEX_WORKER, TEST_INDEX_WORK_INFORMATION))); + } + + @Test + public void testExplainInCorrelatedSubquery() throws IOException { + String expected = loadExpectedPlan("explain_in_correlated_subquery.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source = %s" + + "| where name in [" + + " source = %s | where id = uid and name = 'Tom' | fields name" + + " ]" + + "| sort - salary | fields id, name, salary", + TEST_INDEX_WORKER, TEST_INDEX_WORK_INFORMATION))); + } + + @Test + public void testExplainScalarUncorrelatedSubqueryInSelect() throws IOException { + String expected = loadExpectedPlan("explain_scalar_uncorrelated_subquery_in_select.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source = %s" + + "| eval count_dept = [" + + " source = %s | stats count(name)" + + " ]" + + "| fields name, count_dept", + TEST_INDEX_WORKER, TEST_INDEX_WORK_INFORMATION, TEST_INDEX_WORK_INFORMATION))); + } + + @Test + public void testExplainScalarUncorrelatedSubqueryInWhere() throws IOException { + String expected = loadExpectedPlan("explain_scalar_uncorrelated_subquery_in_where.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source = %s" + + "| where id > [" + + " source = %s | stats count(name)" + + " ] + 999" + + "| fields name", + TEST_INDEX_WORKER, TEST_INDEX_WORK_INFORMATION, TEST_INDEX_WORK_INFORMATION))); + } + + @Test + public void testExplainScalarCorrelatedSubqueryInSelect() throws IOException { + String expected = loadExpectedPlan("explain_scalar_correlated_subquery_in_select.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source = %s" + + "| eval count_dept = [" + + " source = %s" + + " | where id = uid | stats count(name)" + + " ]" + + "| fields id, name, count_dept", + TEST_INDEX_WORKER, TEST_INDEX_WORK_INFORMATION))); + } + + @Test + public void testExplainScalarCorrelatedSubqueryInWhere() throws IOException { + String expected = loadExpectedPlan("explain_scalar_correlated_subquery_in_where.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source = %s" + + "| where id = [" + + " source = %s | where id = uid | stats max(uid)" + + " ]" + + "| fields id, name", + TEST_INDEX_WORKER, TEST_INDEX_WORK_INFORMATION))); } // Only for Calcite @@ -108,9 +241,9 @@ public void supportPushDownSortMergeJoin() throws IOException { String query = "source=opensearch-sql_test_index_bank| join left=l right=r on" + " l.account_number=r.account_number opensearch-sql_test_index_bank"; - var result = explainQueryToString(query); - String expected = loadExpectedPlan("explain_merge_join_sort_push.json"); - assertJsonEqualsIgnoreId(expected, result); + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_merge_join_sort_push.yaml"); + assertYamlEqualsIgnoreId(expected, result); } // Only for Calcite @@ -145,32 +278,54 @@ public void supportPartialPushDown_NoPushIfAllFailed() throws IOException { @Test public void testExplainIsEmpty() throws IOException { // script pushdown - String expected = loadExpectedPlan("explain_isempty.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_isempty.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( - "source=opensearch-sql_test_index_account | where isempty(firstname)")); + explainQueryYaml("source=opensearch-sql_test_index_account | where isempty(firstname)")); + } + + @Test + public void testExplainMultisearchBasic() throws IOException { + String query = + "| multisearch [search" + + " source=opensearch-sql_test_index_account | where age < 30 | eval age_group =" + + " 'young'] [search source=opensearch-sql_test_index_account | where age >= 30 | eval" + + " age_group = 'adult'] | stats count by age_group"; + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_multisearch_basic.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + + @Test + public void testExplainMultisearchTimestampInterleaving() throws IOException { + String query = + "| multisearch " + + "[search source=opensearch-sql_test_index_time_data | where category IN ('A', 'B')] " + + "[search source=opensearch-sql_test_index_time_data2 | where category IN ('E', 'F')] " + + "| head 5"; + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_multisearch_timestamp.yaml"); + assertYamlEqualsIgnoreId(expected, result); } // Only for Calcite @Test public void testExplainIsBlank() throws IOException { // script pushdown - String expected = loadExpectedPlan("explain_isblank.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_isblank.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( - "source=opensearch-sql_test_index_account | where isblank(firstname)")); + explainQueryYaml("source=opensearch-sql_test_index_account | where isblank(firstname)")); } // Only for Calcite @Test public void testExplainIsEmptyOrOthers() throws IOException { // script pushdown - String expected = loadExpectedPlan("explain_isempty_or_others.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_isempty_or_others.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | where gender = 'M' or isempty(firstname) or" + " isnull(firstname)")); } @@ -242,9 +397,9 @@ public void testFilterFunctionScriptPushDownExplain() throws Exception { public void testFilterWithSearchCall() throws IOException { enabledOnlyWhenPushdownIsEnabled(); String expected = loadExpectedPlan("explain_filter_with_search.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format( "source=%s | where birthdate >= '2023-01-01 00:00:00' and birthdate < '2023-01-03" + " 00:00:00' | stats count() by span(birthdate, 1d)", @@ -267,22 +422,56 @@ public void testExplainWithReverse() throws IOException { @Test public void testExplainWithTimechartAvg() throws IOException { - var result = explainQueryToString("source=events | timechart span=1m avg(cpu_usage) by host"); - String expected = - !isPushdownDisabled() - ? loadFromFile("expectedOutput/calcite/explain_timechart.yaml") - : loadFromFile("expectedOutput/calcite/explain_timechart_no_pushdown.yaml"); - assertYamlEqualsJsonIgnoreId(expected, result); + var result = explainQueryYaml("source=events | timechart span=1m avg(cpu_usage) by host"); + String expected = loadExpectedPlan("explain_timechart.yaml"); + assertYamlEqualsIgnoreId(expected, result); } @Test public void testExplainWithTimechartCount() throws IOException { - var result = explainQueryToString("source=events | timechart span=1m count() by host"); - String expected = - !isPushdownDisabled() - ? loadFromFile("expectedOutput/calcite/explain_timechart_count.yaml") - : loadFromFile("expectedOutput/calcite/explain_timechart_count_no_pushdown.yaml"); - assertYamlEqualsJsonIgnoreId(expected, result); + var result = explainQueryYaml("source=events | timechart span=1m count() by host"); + String expected = loadExpectedPlan("explain_timechart_count.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + + @Test + public void testExplainTimechartPerSecond() throws IOException { + var result = explainQueryToString("source=events | timechart span=2m per_second(cpu_usage)"); + assertTrue( + result.contains( + "per_second(cpu_usage)=[DIVIDE(*($1, 1000.0E0), TIMESTAMPDIFF('MILLISECOND':VARCHAR," + + " $0, TIMESTAMPADD('MINUTE':VARCHAR, 2, $0)))]")); + assertTrue(result.contains("per_second(cpu_usage)=[SUM($0)]")); + } + + @Test + public void testExplainTimechartPerMinute() throws IOException { + var result = explainQueryToString("source=events | timechart span=2m per_minute(cpu_usage)"); + assertTrue( + result.contains( + "per_minute(cpu_usage)=[DIVIDE(*($1, 60000.0E0), TIMESTAMPDIFF('MILLISECOND':VARCHAR," + + " $0, TIMESTAMPADD('MINUTE':VARCHAR, 2, $0)))]")); + assertTrue(result.contains("per_minute(cpu_usage)=[SUM($0)]")); + } + + @Test + public void testExplainTimechartPerHour() throws IOException { + var result = explainQueryToString("source=events | timechart span=2m per_hour(cpu_usage)"); + assertTrue( + result.contains( + "per_hour(cpu_usage)=[DIVIDE(*($1, 3600000.0E0), TIMESTAMPDIFF('MILLISECOND':VARCHAR," + + " $0, TIMESTAMPADD('MINUTE':VARCHAR, 2, $0)))]")); + assertTrue(result.contains("per_hour(cpu_usage)=[SUM($0)]")); + } + + @Test + public void testExplainTimechartPerDay() throws IOException { + var result = explainQueryToString("source=events | timechart span=2m per_day(cpu_usage)"); + assertTrue( + result.contains( + "per_day(cpu_usage)=[DIVIDE(*($1, 8.64E7), TIMESTAMPDIFF('MILLISECOND':VARCHAR, $0," + + " TIMESTAMPADD('MINUTE':VARCHAR, 2, $0)))]")); + assertTrue(result.contains("per_day(cpu_usage)=[SUM($0)]")); } @Test @@ -291,9 +480,9 @@ public void noPushDownForAggOnWindow() throws IOException { String query = "source=opensearch-sql_test_index_account | patterns address method=BRAIN | stats count()" + " by patterns_field"; - var result = explainQueryToString(query); - String expected = loadFromFile("expectedOutput/calcite/explain_agg_on_window.json"); - assertJsonEqualsIgnoreId(expected, result); + var result = explainQueryYaml(query); + String expected = loadFromFile("expectedOutput/calcite/explain_agg_on_window.yaml"); + assertYamlEqualsIgnoreId(expected, result); } // Only for Calcite @@ -301,11 +490,11 @@ public void noPushDownForAggOnWindow() throws IOException { public void supportPushDownScriptOnTextField() throws IOException { enabledOnlyWhenPushdownIsEnabled(); String result = - explainQueryToString( + explainQueryYaml( "explain source=opensearch-sql_test_index_account | where length(address) > 0 | eval" + " address_length = length(address) | stats count() by address_length"); - String expected = loadFromFile("expectedOutput/calcite/explain_script_push_on_text.json"); - assertJsonEqualsIgnoreId(expected, result); + String expected = loadFromFile("expectedOutput/calcite/explain_script_push_on_text.yaml"); + assertYamlEqualsIgnoreId(expected, result); } @Test @@ -321,25 +510,41 @@ public void testExplainStatsWithBinsOnTimeField() throws IOException { // TODO: Remove this after addressing https://github.com/opensearch-project/sql/issues/4317 enabledOnlyWhenPushdownIsEnabled(); String expected = loadExpectedPlan("explain_stats_bins_on_time.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( - "source=events | bin @timestamp bins=3 | stats count() by @timestamp")); + explainQueryYaml("source=events | bin @timestamp bins=3 | stats count() by @timestamp")); expected = loadExpectedPlan("explain_stats_bins_on_time2.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=events | bin @timestamp bins=3 | stats avg(cpu_usage) by @timestamp")); } + @Test + public void testExplainStatsWithSubAggregation() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String expected = loadExpectedPlan("explain_stats_bins_on_time_and_term.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + "source=events | bin @timestamp bins=3 | stats bucket_nullable=false count() by" + + " @timestamp, region")); + + expected = loadExpectedPlan("explain_stats_bins_on_time_and_term2.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + "source=events | bin @timestamp bins=3 | stats bucket_nullable=false avg(cpu_usage) by" + + " @timestamp, region")); + } + @Test public void testExplainBinWithSpan() throws IOException { String expected = loadExpectedPlan("explain_bin_span.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( - "source=opensearch-sql_test_index_account | bin age span=10 | head 5")); + explainQueryYaml("source=opensearch-sql_test_index_account | bin age span=10 | head 5")); } @Test @@ -363,9 +568,9 @@ public void testExplainBinWithStartEnd() throws IOException { @Test public void testExplainBinWithAligntime() throws IOException { String expected = loadExpectedPlan("explain_bin_aligntime.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_time_data | bin @timestamp span=2h aligntime=latest |" + " head 5")); } @@ -410,19 +615,71 @@ public void testEventstatsDistinctCountFunctionExplain() throws IOException { assertJsonEqualsIgnoreId(expected, result); } + @Test + public void testStreamstatsDistinctCountExplain() throws IOException { + String query = + "source=opensearch-sql_test_index_account | streamstats dc(state) as distinct_states"; + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_streamstats_dc.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + + @Test + public void testStreamstatsDistinctCountFunctionExplain() throws IOException { + String query = + "source=opensearch-sql_test_index_account | streamstats distinct_count(state) as" + + " distinct_states by gender"; + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_streamstats_distinct_count.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + + @Test + public void testStreamstatsGlobalExplain() throws IOException { + String query = + "source=opensearch-sql_test_index_account | streamstats window=2 global=true avg(age) as" + + " avg_age by gender"; + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_streamstats_global.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + + @Test + public void testStreamstatsResetExplain() throws IOException { + String query = + "source=opensearch-sql_test_index_account | streamstats current=false reset_before=age>34" + + " reset_after=age<25 avg(age) as avg_age by gender"; + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_streamstats_reset.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + // Only for Calcite, as v2 gets unstable serialized string for function @Test public void testExplainOnAggregationWithSumEnhancement() throws IOException { String expected = loadExpectedPlan("explain_agg_with_sum_enhancement.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format( "source=%s | stats sum(balance), sum(balance + 100), sum(balance - 100)," + " sum(balance * 100), sum(balance / 100) by gender", TEST_INDEX_BANK))); } + @Test + public void testStatsDistinctCountApproxFunctionExplainWithPushDown() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String query = + "source=opensearch-sql_test_index_account | stats distinct_count_approx(state) as" + + " distinct_states by gender"; + var result = explainQueryToString(query); + String expected = + loadFromFile( + "expectedOutput/calcite/explain_agg_with_distinct_count_approx_enhancement.json"); + assertJsonEqualsIgnoreId(expected, result); + } + @Test public void testExplainRegexMatchInWhereWithScriptPushdown() throws IOException { enabledOnlyWhenPushdownIsEnabled(); @@ -448,10 +705,10 @@ public void testExplainRegexMatchInEvalWithOutScriptPushdown() throws IOExceptio // Only for Calcite @Test public void testExplainOnEarliestLatest() throws IOException { - String expected = loadExpectedPlan("explain_earliest_latest.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_earliest_latest.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format( "source=%s | stats earliest(message) as earliest_message, latest(message) as" + " latest_message by server", @@ -461,10 +718,10 @@ public void testExplainOnEarliestLatest() throws IOException { // Only for Calcite @Test public void testExplainOnEarliestLatestWithCustomTimeField() throws IOException { - String expected = loadExpectedPlan("explain_earliest_latest_custom_time.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_earliest_latest_custom_time.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format( "source=%s | stats earliest(message, created_at) as earliest_message," + " latest(message, created_at) as latest_message by level", @@ -474,10 +731,10 @@ public void testExplainOnEarliestLatestWithCustomTimeField() throws IOException // Only for Calcite @Test public void testExplainOnFirstLast() throws IOException { - String expected = loadExpectedPlan("explain_first_last.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_first_last.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format( "source=%s | stats first(firstname) as first_name, last(firstname) as" + " last_name by gender", @@ -522,6 +779,41 @@ public void testExplainOnEventstatsEarliestLatestNoGroupBy() throws IOException TEST_INDEX_LOGS))); } + public void testExplainOnStreamstatsEarliestLatest() throws IOException { + String expected = loadExpectedPlan("explain_streamstats_earliest_latest.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source=%s | streamstats earliest(message) as earliest_message, latest(message) as" + + " latest_message by server", + TEST_INDEX_LOGS))); + } + + @Test + public void testExplainOnStreamstatsEarliestLatestWithCustomTimeField() throws IOException { + String expected = loadExpectedPlan("explain_streamstats_earliest_latest_custom_time.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source=%s | streamstats earliest(message, created_at) as earliest_message," + + " latest(message, created_at) as latest_message by level", + TEST_INDEX_LOGS))); + } + + @Test + public void testExplainOnStreamstatsEarliestLatestNoGroupBy() throws IOException { + String expected = loadExpectedPlan("explain_streamstats_earliest_latest_no_group.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source=%s | streamstats earliest(message) as earliest_message, latest(message) as" + + " latest_message", + TEST_INDEX_LOGS))); + } + @Test public void testListAggregationExplain() throws IOException { String expected = loadExpectedPlan("explain_list_aggregation.json"); @@ -544,17 +836,17 @@ public void testValuesAggregationExplain() throws IOException { public void testRegexExplain() throws IOException { String query = "source=opensearch-sql_test_index_account | regex lastname='^[A-Z][a-z]+$' | head 5"; - var result = explainQueryToString(query); + var result = explainQueryYaml(query); String expected = loadExpectedPlan("explain_regex.yaml"); - assertYamlEqualsJsonIgnoreId(expected, result); + assertYamlEqualsIgnoreId(expected, result); } @Test public void testRegexNegatedExplain() throws IOException { String query = "source=opensearch-sql_test_index_account | regex lastname!='.*son$' | head 5"; - var result = explainQueryToString(query); + var result = explainQueryYaml(query); String expected = loadExpectedPlan("explain_regex_negated.yaml"); - assertYamlEqualsJsonIgnoreId(expected, result); + assertYamlEqualsIgnoreId(expected, result); } @Test @@ -580,9 +872,9 @@ public void testRexExplain() throws IOException { String query = "source=opensearch-sql_test_index_account | rex field=lastname \\\"(?^[A-Z])\\\" |" + " head 5"; - var result = explainQueryToString(query); + var result = explainQueryYaml(query); String expected = loadExpectedPlan("explain_rex.yaml"); - assertYamlEqualsJsonIgnoreId(expected, result); + assertYamlEqualsIgnoreId(expected, result); } @Test @@ -614,9 +906,9 @@ public void testPreventLimitPushdown() throws IOException { enabledOnlyWhenPushdownIsEnabled(); setMaxResultWindow("opensearch-sql_test_index_account", 1); String query = "source=opensearch-sql_test_index_account | head 1 from 1"; - var result = explainQueryToString(query); + var result = explainQueryYaml(query); String expected = loadExpectedPlan("explain_prevent_limit_push.yaml"); - assertYamlEqualsJsonIgnoreId(expected, result); + assertYamlEqualsIgnoreId(expected, result); resetMaxResultWindow("opensearch-sql_test_index_account"); } @@ -629,9 +921,9 @@ public void testPushdownLimitIntoAggregation() throws IOException { explainQueryToString("source=opensearch-sql_test_index_account | stats count() by state")); expected = loadExpectedPlan("explain_limit_agg_pushdown2.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | stats count() by state | head 100")); expected = loadExpectedPlan("explain_limit_agg_pushdown3.json"); @@ -641,24 +933,24 @@ public void testPushdownLimitIntoAggregation() throws IOException { "source=opensearch-sql_test_index_account | stats count() by state | head 100 | head 10" + " from 10 ")); - expected = loadExpectedPlan("explain_limit_agg_pushdown4.json"); - assertJsonEqualsIgnoreId( + expected = loadExpectedPlan("explain_limit_agg_pushdown4.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | stats count() by state | sort state | head" + " 100 | head 10 from 10 ")); - expected = loadExpectedPlan("explain_limit_agg_pushdown_bucket_nullable1.json"); - assertJsonEqualsIgnoreId( + expected = loadExpectedPlan("explain_limit_agg_pushdown_bucket_nullable1.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | stats bucket_nullable=false count() by" + " state | head 100 | head 10 from 10 ")); - expected = loadExpectedPlan("explain_limit_agg_pushdown_bucket_nullable2.json"); - assertJsonEqualsIgnoreId( + expected = loadExpectedPlan("explain_limit_agg_pushdown_bucket_nullable2.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | stats bucket_nullable=false count() by" + " state | sort state | head 100 | head 10 from 10 ")); @@ -673,18 +965,18 @@ public void testPushdownLimitIntoAggregation() throws IOException { @Test public void testExplainMaxOnStringField() throws IOException { - String expected = loadExpectedPlan("explain_max_string_field.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_max_string_field.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString("source=opensearch-sql_test_index_account | stats max(firstname)")); + explainQueryYaml("source=opensearch-sql_test_index_account | stats max(firstname)")); } @Test public void testExplainMinOnStringField() throws IOException { - String expected = loadExpectedPlan("explain_min_string_field.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_min_string_field.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString("source=opensearch-sql_test_index_account | stats min(firstname)")); + explainQueryYaml("source=opensearch-sql_test_index_account | stats min(firstname)")); } @Test @@ -693,75 +985,75 @@ public void testCountAggPushDownExplain() throws IOException { enabledOnlyWhenPushdownIsEnabled(); // should be optimized by hits.total.value String expected = loadExpectedPlan("explain_count_agg_push1.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString("source=opensearch-sql_test_index_account | stats count() as cnt")); + explainQueryYaml("source=opensearch-sql_test_index_account | stats count() as cnt")); // should be optimized expected = loadExpectedPlan("explain_count_agg_push2.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | stats count(lastname) as cnt")); // should be optimized expected = loadExpectedPlan("explain_count_agg_push3.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | eval name = lastname | stats count(name) as" + " cnt")); // should be optimized expected = loadExpectedPlan("explain_count_agg_push4.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | stats count() as c1, count() as c2")); // should be optimized expected = loadExpectedPlan("explain_count_agg_push5.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | stats count(lastname) as c1," + " count(lastname) as c2")); // should be optimized expected = loadExpectedPlan("explain_count_agg_push6.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | eval name = lastname | stats" + " count(lastname), count(name)")); // should not be optimized expected = loadExpectedPlan("explain_count_agg_push7.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | stats count(balance + 1) as cnt")); // should not be optimized expected = loadExpectedPlan("explain_count_agg_push8.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | stats count() as c1, count(lastname) as" + " c2")); // should not be optimized expected = loadExpectedPlan("explain_count_agg_push9.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | stats count(firstname), count(lastname)")); // should not be optimized expected = loadExpectedPlan("explain_count_agg_push10.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | eval name = lastname | stats" + " count(firstname), count(name)")); } @@ -771,26 +1063,26 @@ public void testExplainCountsByAgg() throws IOException { enabledOnlyWhenPushdownIsEnabled(); String expected = loadExpectedPlan("explain_agg_counts_by1.yaml"); // case of only count(): doc_count works - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format( "source=%s | stats count(), count() as c1 by gender", TEST_INDEX_ACCOUNT))); // count(FIELD) by: doc_count doesn't work expected = loadExpectedPlan("explain_agg_counts_by2.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format( "source=%s | stats count(balance) as c1, count(balance) as c2 by gender", TEST_INDEX_ACCOUNT))); // count(FIELD) by: doc_count doesn't work expected = loadExpectedPlan("explain_agg_counts_by3.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format( "source=%s | eval account_number_alias = account_number" + " | stats count(account_number), count(account_number_alias) as c2 by gender", @@ -798,26 +1090,26 @@ public void testExplainCountsByAgg() throws IOException { // count() + count(FIELD)): doc_count doesn't work expected = loadExpectedPlan("explain_agg_counts_by4.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format( "source=%s | stats count(), count(account_number) by gender", TEST_INDEX_ACCOUNT))); // count(FIELD1) + count(FIELD2)) by: doc_count doesn't work expected = loadExpectedPlan("explain_agg_counts_by5.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format( "source=%s | stats count(balance), count(account_number) by gender", TEST_INDEX_ACCOUNT))); // case of count(EXPRESSION) by: doc_count doesn't work expected = loadExpectedPlan("explain_agg_counts_by6.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format( "source=%s | eval b_1 = balance + 1" + " | stats count(b_1), count(pow(balance, 2)) as c3 by gender", @@ -825,24 +1117,118 @@ public void testExplainCountsByAgg() throws IOException { } @Test - public void testExplainSortOnMetricsNoBucketNullable() throws IOException { - // TODO enhancement later: https://github.com/opensearch-project/sql/issues/4282 + public void testExplainSortOnMeasure() throws IOException { enabledOnlyWhenPushdownIsEnabled(); - String expected = loadExpectedPlan("explain_agg_sort_on_metrics1.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_agg_sort_on_measure1.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | stats bucket_nullable=false count() by" + " state | sort `count()`")); + expected = loadExpectedPlan("explain_agg_sort_on_measure2.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + "source=opensearch-sql_test_index_account | stats bucket_nullable=false sum(balance)" + + " as sum by state | sort - sum")); + // TODO limit should pushdown to non-composite agg + expected = loadExpectedPlan("explain_agg_sort_on_measure3.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source=%s | stats count() as cnt by span(birthdate, 1d) | sort - cnt", + TEST_INDEX_BANK))); + expected = loadExpectedPlan("explain_agg_sort_on_measure4.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source=%s | stats bucket_nullable=false sum(balance) by span(age, 5) | sort -" + + " `sum(balance)`", + TEST_INDEX_BANK))); + } - expected = loadExpectedPlan("explain_agg_sort_on_metrics2.json"); - assertJsonEqualsIgnoreId( + @Test + public void testExplainSortOnMeasureMultiTerms() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String expected = loadExpectedPlan("explain_agg_sort_on_measure_multi_terms.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | stats bucket_nullable=false count() by" + " gender, state | sort `count()`")); } + @Test + public void testExplainCompositeMultiBucketsAutoDateThenSortOnMeasureNotPushdown() + throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + assertYamlEqualsIgnoreId( + loadExpectedPlan("agg_composite_multi_terms_autodate_sort_agg_measure_not_push.yaml"), + explainQueryYaml( + String.format( + "source=%s | bin timestamp bins=3 | stats bucket_nullable=false avg(value), count()" + + " as cnt by category, value, timestamp | sort cnt", + TEST_INDEX_TIME_DATA))); + } + + @Test + public void testExplainCompositeRangeThenSortOnMeasureNotPushdown() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + assertYamlEqualsIgnoreId( + loadExpectedPlan("agg_composite_range_sort_agg_measure_not_push.yaml"), + explainQueryYaml( + String.format( + "source=%s | eval value_range = case(value < 7000, 'small'" + + " else 'great') | stats bucket_nullable=false avg(value), count() as cnt by" + + " value_range, category | sort cnt", + TEST_INDEX_TIME_DATA))); + } + + @Test + public void testExplainCompositeAutoDateThenSortOnMeasureNotPushdown() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + assertYamlEqualsIgnoreId( + loadExpectedPlan("agg_composite_autodate_sort_agg_measure_not_push.yaml"), + explainQueryYaml( + String.format( + "source=%s | bin timestamp bins=3 | stats bucket_nullable=false avg(value), count()" + + " as cnt by timestamp, category | sort cnt", + TEST_INDEX_TIME_DATA))); + } + + @Test + public void testExplainCompositeRangeAutoDateThenSortOnMeasureNotPushdown() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + assertYamlEqualsIgnoreId( + loadExpectedPlan("agg_composite_autodate_range_metric_sort_agg_measure_not_push.yaml"), + explainQueryYaml( + String.format( + "source=%s | bin timestamp bins=3 | eval value_range = case(value < 7000, 'small'" + + " else 'great') | stats bucket_nullable=false avg(value), count() as cnt by" + + " timestamp, value_range, category | sort cnt", + TEST_INDEX_TIME_DATA))); + } + + @Test + public void testExplainMultipleAggregatorsWithSortOnOneMeasureNotPushDown() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String expected = + loadExpectedPlan("explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + "source=opensearch-sql_test_index_account | stats bucket_nullable=false count() as c," + + " sum(balance) as s by state | sort c")); + expected = loadExpectedPlan("explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + "source=opensearch-sql_test_index_account | stats bucket_nullable=false count() as c," + + " sum(balance) as s by state | sort c, s")); + } + @Test public void testExplainEvalMax() throws IOException { String expected = loadExpectedPlan("explain_eval_max.json"); @@ -895,20 +1281,294 @@ public void testExplainPushDownScriptsContainingUDT() throws IOException { "source=%s | where cidrmatch(host, '0.0.0.0/24') | fields host", TEST_INDEX_WEBLOGS))); - assertJsonEqualsIgnoreId( - loadExpectedPlan("explain_agg_script_timestamp_push.json"), - explainQueryToString( + assertYamlEqualsIgnoreId( + loadExpectedPlan("explain_agg_script_timestamp_push.yaml"), + explainQueryYaml( String.format( "source=%s | eval t = unix_timestamp(birthdate) | stats count() by t | sort t |" + " head 3", TEST_INDEX_BANK))); - assertJsonEqualsIgnoreId( - loadExpectedPlan("explain_agg_script_udt_arg_push.json"), - explainQueryToString( + assertYamlEqualsIgnoreId( + loadExpectedPlan("explain_agg_script_udt_arg_push.yaml"), + explainQueryYaml( String.format( "source=%s | eval t = date_add(birthdate, interval 1 day) | stats count() by" + " span(t, 1d)", TEST_INDEX_BANK))); } + + @Test + public void testFillNullValueSyntaxExplain() throws IOException { + String expected = loadExpectedPlan("explain_fillnull_value_syntax.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source=%s | fields age, balance | fillnull value=0", TEST_INDEX_ACCOUNT))); + } + + @Test + public void testJoinWithPushdownSortIntoAgg() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + // PPL_JOIN_SUBSEARCH_MAXOUT!=0 will add limit before sort and then prevent sort push down. + setJoinSubsearchMaxOut(0); + String expected = loadExpectedPlan("explain_join_with_agg.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source=%s | stats COUNT() by age, gender | join left=L right=R ON L.gender =" + + " R.gender [source=%s | stats COUNT() as overall_cnt by gender]", + TEST_INDEX_ACCOUNT, TEST_INDEX_ACCOUNT))); + resetJoinSubsearchMaxOut(); + } + + @Test + public void testReplaceCommandExplain() throws IOException { + String expected = loadExpectedPlan("explain_replace_command.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source=%s | replace 'IL' WITH 'Illinois' IN state | fields state", + TEST_INDEX_ACCOUNT))); + } + + @Test + public void testReplaceCommandWildcardExplain() throws IOException { + String expected = loadExpectedPlan("explain_replace_wildcard.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source=%s | replace '*L' WITH 'STATE_IL' IN state | fields state", + TEST_INDEX_ACCOUNT))); + } + + @Test + public void testExplainRareCommandUseNull() throws IOException { + String expected = loadExpectedPlan("explain_rare_usenull_false.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format("source=%s | rare 2 usenull=false state by gender", TEST_INDEX_ACCOUNT))); + expected = loadExpectedPlan("explain_rare_usenull_true.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format("source=%s | rare 2 usenull=true state by gender", TEST_INDEX_ACCOUNT))); + withSettings( + Settings.Key.PPL_SYNTAX_LEGACY_PREFERRED, + "false", + () -> { + try { + assertYamlEqualsIgnoreId( + loadExpectedPlan("explain_rare_usenull_false.yaml"), + explainQueryYaml( + String.format("source=%s | rare 2 state by gender", TEST_INDEX_ACCOUNT))); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + @Test + public void testExplainTopCommandUseNull() throws IOException { + String expected = loadExpectedPlan("explain_top_usenull_false.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format("source=%s | top 2 usenull=false state by gender", TEST_INDEX_ACCOUNT))); + expected = loadExpectedPlan("explain_top_usenull_true.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format("source=%s | top 2 usenull=true state by gender", TEST_INDEX_ACCOUNT))); + withSettings( + Settings.Key.PPL_SYNTAX_LEGACY_PREFERRED, + "false", + () -> { + try { + assertYamlEqualsIgnoreId( + loadExpectedPlan("explain_top_usenull_false.yaml"), + explainQueryYaml( + String.format("source=%s | top 2 state by gender", TEST_INDEX_ACCOUNT))); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + // Test cases for verifying the fix of https://github.com/opensearch-project/sql/issues/4571 + @Test + public void testPushDownMinOrMaxAggOnDerivedField() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String expected = loadExpectedPlan("explain_min_max_agg_on_derived_field.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source=%s | eval balance2 = CEIL(balance/10000.0) " + + "| stats MIN(balance2), MAX(balance2)", + TEST_INDEX_ACCOUNT))); + } + + @Test + public void testCasePushdownAsRangeQueryExplain() throws IOException { + // CASE 1: Range - Metric + // 1.1 Range - Metric + assertYamlEqualsIgnoreId( + loadExpectedPlan("agg_range_metric_push.yaml"), + explainQueryYaml( + String.format( + "source=%s | eval age_range = case(age < 30, 'u30', age < 40, 'u40' else 'u100') |" + + " stats avg(age) as avg_age by age_range", + TEST_INDEX_BANK))); + + // 1.2 Range - Metric (COUNT) + assertYamlEqualsIgnoreId( + loadExpectedPlan("agg_range_count_push.yaml"), + explainQueryYaml( + String.format( + "source=%s | eval age_range = case(age < 30, 'u30', age >= 30 and age < 40, 'u40'" + + " else 'u100') | stats avg(age) by age_range", + TEST_INDEX_BANK))); + + // 1.3 Range - Range - Metric + assertYamlEqualsIgnoreId( + loadExpectedPlan("agg_range_range_metric_push.yaml"), + explainQueryYaml( + String.format( + "source=%s | eval age_range = case(age < 30, 'u30', age < 40, 'u40' else 'u100')," + + " balance_range = case(balance < 20000, 'medium' else 'high') | stats" + + " avg(balance) as avg_balance by age_range, balance_range", + TEST_INDEX_BANK))); + + // 1.4 Range - Metric (With null & discontinuous ranges) + assertYamlEqualsIgnoreId( + loadExpectedPlan("agg_range_metric_complex_push.yaml"), + explainQueryYaml( + String.format( + "source=%s | eval age_range = case(age < 30, 'u30', (age >= 35 and age < 40) or age" + + " >= 80, '30-40 or >=80') | stats avg(balance) by age_range", + TEST_INDEX_BANK))); + + // 1.5 Should not be pushed because the range is not closed-open + assertYamlEqualsIgnoreId( + loadExpectedPlan("agg_case_cannot_push.yaml"), + explainQueryYaml( + String.format( + "source=%s | eval age_range = case(age < 30, 'u30', age >= 30 and age <= 40, 'u40'" + + " else 'u100') | stats avg(age) as avg_age by age_range", + TEST_INDEX_BANK))); + + // 1.6 Should not be pushed as range query because the result expression is not a string + // literal. + // Range aggregation keys must be strings + assertYamlEqualsIgnoreId( + loadExpectedPlan("agg_case_num_res_cannot_push.yaml"), + explainQueryYaml( + String.format( + "source=%s | eval age_range = case(age < 30, 30 else 100) | stats count() by" + + " age_range", + TEST_INDEX_BANK))); + + // CASE 2: Composite - Range - Metric + // 2.1 Composite (term) - Range - Metric + assertYamlEqualsIgnoreId( + loadExpectedPlan("agg_composite_range_metric_push.yaml"), + explainQueryYaml( + String.format( + "source=%s | eval age_range = case(age < 30, 'u30' else 'a30') | stats avg(balance)" + + " by state, age_range", + TEST_INDEX_BANK))); + + // 2.2 Composite (date histogram) - Range - Metric + assertYamlEqualsIgnoreId( + loadExpectedPlan("agg_composite_date_range_push.yaml"), + explainQueryYaml( + "source=opensearch-sql_test_index_time_data | eval value_range = case(value < 7000," + + " 'small' else 'large') | stats avg(value) by value_range, span(@timestamp," + + " 1h)")); + + // 2.3 Composite(2 fields) - Range - Metric (with count) + assertYamlEqualsIgnoreId( + loadExpectedPlan("agg_composite2_range_count_push.yaml"), + explainQueryYaml( + String.format( + "source=%s | eval age_range = case(age < 30, 'u30' else 'a30') | stats" + + " avg(balance), count() by age_range, state, gender", + TEST_INDEX_BANK))); + + // 2.4 Composite (2 fields) - Range - Range - Metric (with count) + assertYamlEqualsIgnoreId( + loadExpectedPlan("agg_composite2_range_range_count_push.yaml"), + explainQueryYaml( + String.format( + "source=%s | eval age_range = case(age < 35, 'u35' else 'a35'), balance_range =" + + " case(balance < 20000, 'medium' else 'high') | stats avg(balance) as" + + " avg_balance by age_range, balance_range, state", + TEST_INDEX_BANK))); + + // 2.5 Should not be pushed down as range query because case result expression is not constant + assertYamlEqualsIgnoreId( + loadExpectedPlan("agg_case_composite_cannot_push.yaml"), + explainQueryYaml( + String.format( + "source=%s | eval age_range = case(age < 35, 'u35' else email) | stats avg(balance)" + + " as avg_balance by age_range, state", + TEST_INDEX_BANK))); + } + + @Test + public void testNestedAggregationsExplain() throws IOException { + // TODO: Remove after resolving: https://github.com/opensearch-project/sql/issues/4578 + enabledOnlyWhenPushdownIsEnabled(); + assertYamlEqualsIgnoreId( + loadExpectedPlan("agg_composite_autodate_range_metric_push.yaml"), + explainQueryYaml( + String.format( + "source=%s | bin timestamp bins=3 | eval value_range = case(value < 7000, 'small'" + + " else 'great') | stats bucket_nullable=false avg(value), count() by" + + " timestamp, value_range, category", + TEST_INDEX_TIME_DATA))); + } + + @Test + public void testTopKThenSortExplain() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String expected = loadExpectedPlan("explain_top_k_then_sort_push.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + "source=opensearch-sql_test_index_account" + + "| sort balance" + + "| head 5 " + + "| sort age " + + "| fields age")); + } + + @Test + public void testGeoIpPushedInAgg() throws IOException { + // This explain IT verifies that externally registered UDF can be properly pushed down + assertYamlEqualsIgnoreId( + loadExpectedPlan("udf_geoip_in_agg_pushed.yaml"), + explainQueryYaml( + String.format( + "source=%s | eval info = geoip('my-datasource', host) | stats count() by info.city", + TEST_INDEX_WEBLOGS))); + } + + @Test + public void testInternalItemAccessOnStructs() throws IOException { + String expected = loadExpectedPlan("access_struct_subfield_with_item.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source=%s | eval info = geoip('dummy-datasource', host) | fields host, info," + + " info.dummy_sub_field", + TEST_INDEX_WEBLOGS))); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFillNullCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFillNullCommandIT.java index 08b9da7b1ad..9fb1a96046f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFillNullCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFillNullCommandIT.java @@ -5,6 +5,14 @@ package org.opensearch.sql.calcite.remote; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CALCS; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifyErrorMessageContains; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; import org.opensearch.sql.ppl.FillNullCommandIT; public class CalciteFillNullCommandIT extends FillNullCommandIT { @@ -13,4 +21,157 @@ public void init() throws Exception { super.init(); enableCalcite(); } + + @Test + public void testFillNullValueSyntaxAllFields() throws IOException { + // fillnull value=0 (applies to all fields) + JSONObject result = + executeQuery( + String.format("source=%s | fields num0, num2 | fillnull value=0", TEST_INDEX_CALCS)); + verifyDataRows( + result, + rows(12.3, 17.86), + rows(-12.3, 16.73), + rows(15.7, 0), + rows(-15.7, 8.51), + rows(3.5, 6.46), + rows(-3.5, 8.98), + rows(0, 11.69), + rows(0, 17.25), + rows(10, 0), + rows(0, 11.5), + rows(0, 6.8), + rows(0, 3.79), + rows(0, 0), + rows(0, 13.04), + rows(0, 0), + rows(0, 10.98), + rows(0, 7.87)); + } + + @Test + public void testFillNullValueSyntaxWithFields() throws IOException { + // fillnull value=-1 num0 (applies to specified field only) + JSONObject result = + executeQuery( + String.format( + "source=%s | fields str2, num0 | fillnull value=-1 num0", TEST_INDEX_CALCS)); + verifyDataRows( + result, + rows("one", 12.3), + rows("two", -12.3), + rows("three", 15.7), + rows(null, -15.7), + rows("five", 3.5), + rows("six", -3.5), + rows(null, 0), + rows("eight", -1), + rows("nine", 10), + rows("ten", -1), + rows("eleven", -1), + rows("twelve", -1), + rows(null, -1), + rows("fourteen", -1), + rows("fifteen", -1), + rows("sixteen", -1), + rows(null, -1)); + } + + @Test + public void testFillNullValueSyntaxWithStringValue() throws IOException { + // fillnull value='N/A' str2 (string replacement value) + JSONObject result = + executeQuery( + String.format( + "source=%s | fields str2, int0 | fillnull value='N/A' str2", TEST_INDEX_CALCS)); + verifyDataRows( + result, + rows("one", 1), + rows("two", null), + rows("three", null), + rows("N/A", null), + rows("five", 7), + rows("six", 3), + rows("N/A", 8), + rows("eight", null), + rows("nine", null), + rows("ten", 8), + rows("eleven", 4), + rows("twelve", 10), + rows("N/A", null), + rows("fourteen", 4), + rows("fifteen", 11), + rows("sixteen", 4), + rows("N/A", 8)); + } + + // Type restriction error tests - documents error messages when type mismatches occur + + @Test + public void testFillNullWithMixedTypeFieldsError() { + // fillnull value=0 on mixed type fields without explicit field list + // When no fields specified, all fields must be same type + Throwable t = + assertThrowsWithReplace( + RuntimeException.class, + () -> + executeQuery( + String.format( + "source=%s | fields str2, int0 | fillnull value=0", TEST_INDEX_CALCS))); + verifyErrorMessageContains( + t, + "fillnull failed: replacement value type INTEGER is not compatible with field 'str2' " + + "(type: VARCHAR). The replacement value type must match the field type."); + } + + @Test + public void testFillNullWithStringOnNumericAndStringMixedFields() { + // fillnull value='test' applied to fields of different types + // Trying to fill both string and numeric fields with a string value + Throwable t = + assertThrowsWithReplace( + RuntimeException.class, + () -> + executeQuery( + String.format( + "source=%s | fields num0, str2 | fillnull value='test' num0 str2", + TEST_INDEX_CALCS))); + + System.out.println("Debugging error message: " + t); + verifyErrorMessageContains( + t, + "fillnull failed: replacement value type VARCHAR is not compatible with field 'num0' " + + "(type: DOUBLE). The replacement value type must match the field type."); + } + + @Test + public void testFillNullWithLargeIntegerOnIntField() throws IOException { + // fillnull using int0=8589934592 (2^33, larger than Integer.MAX_VALUE) + // Tests whether type family equality allows BIGINT value for INTEGER field + JSONObject result = + executeQuery( + String.format( + "source=%s | fields int0 | fillnull using int0=8589934592", TEST_INDEX_CALCS)); + // If this test passes, it confirms that NUMERIC type family allows INT->BIGINT coercion + // The result should show 8589934592 for null int0 values + verifyDataRows( + result, + rows(1), + rows(8589934592L), + rows(8589934592L), + rows(8589934592L), + rows(7), + rows(3), + rows(8), + rows(8589934592L), + rows(8589934592L), + rows(8), + rows(4), + rows(10), + rows(8589934592L), + rows(4), + rows(11), + rows(4), + rows(8)); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteGeoIpFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteGeoIpFunctionsIT.java index 9de93a515a2..3ca0bfae4ce 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteGeoIpFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteGeoIpFunctionsIT.java @@ -5,12 +5,102 @@ package org.opensearch.sql.calcite.remote; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOGS; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import java.io.IOException; +import java.util.Map; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; import org.opensearch.sql.ppl.GeoIpFunctionsIT; public class CalciteGeoIpFunctionsIT extends GeoIpFunctionsIT { @Override public void init() throws Exception { super.init(); + loadIndex(Index.WEBLOG); enableCalcite(); + + // Only limited IPs are loaded into geospatial data sources. Therefore, we insert IPs that match + // those known ones for test purpose + Request bulkRequest = new Request("POST", "/_bulk?refresh=true"); + bulkRequest.setJsonEntity( + String.format( + "{\"index\":{\"_index\":\"%s\",\"_id\":6}}\n" + + "{\"host\":\"10.0.0.1\",\"method\":\"POST\"}\n" + + "{\"index\":{\"_index\":\"%s\",\"_id\":7}}\n" + + "{\"host\":\"fd12:2345:6789:1:a1b2:c3d4:e5f6:789a\",\"method\":\"POST\"}\n", + TEST_INDEX_WEBLOGS, TEST_INDEX_WEBLOGS)); + client().performRequest(bulkRequest); + } + + // In v2 it supports only string as IP inputs + @Test + public void testGeoIpEnrichmentWithIpFieldAsInput() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | where method='POST' | eval ip_to_country = geoip('%s', host," + + " 'country') | fields host, ip_to_country", + TEST_INDEX_WEBLOGS, DATASOURCE_NAME)); + verifySchema(result, schema("host", "ip"), schema("ip_to_country", "struct")); + verifyDataRows( + result, + rows("10.0.0.1", Map.of("country", "USA")), + rows("fd12:2345:6789:1:a1b2:c3d4:e5f6:789a", Map.of("country", "India"))); + } + + @Test + public void testGeoIpInAggregation() throws IOException { + JSONObject result1 = + executeQuery( + String.format( + "source=%s | where method='POST' | eval info = geoip('%s', host) | eval" + + " date=DATE('2020-12-10') | stats count() by info.city, method, span(date," + + " 1month) as month", + TEST_INDEX_WEBLOGS, DATASOURCE_NAME)); + verifySchema( + result1, + schema("count()", "bigint"), + schema("month", "date"), + schema("info.city", "string"), + schema("method", "string")); + verifyDataRows( + result1, + rows(1, "2020-12-01", "Seattle", "POST"), + rows(1, "2020-12-01", "Bengaluru", "POST")); + + // This case is pushed down into DSL with scripts + JSONObject result2 = + executeQuery( + String.format( + "source=%s | where method='POST' | eval info = geoip('%s', host) | stats count() by" + + " info.city", + TEST_INDEX_WEBLOGS, DATASOURCE_NAME)); + verifySchema(result2, schema("count()", "bigint"), schema("info.city", "string")); + verifyDataRows(result2, rows(1, "Seattle"), rows(1, "Bengaluru")); + } + + @Test + public void testGeoIpEnrichmentAccessingSubField() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | where method='POST' | eval info = geoip('%s', host) | fields host," + + " info, info.country", + TEST_INDEX_WEBLOGS, DATASOURCE_NAME)); + verifySchema( + result, schema("host", "ip"), schema("info", "struct"), schema("info.country", "string")); + verifyDataRows( + result, + rows("10.0.0.1", Map.of("country", "USA", "city", "Seattle"), "USA"), + rows( + "fd12:2345:6789:1:a1b2:c3d4:e5f6:789a", + Map.of("country", "India", "city", "Bengaluru"), + "India")); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMVAppendFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMVAppendFunctionIT.java new file mode 100644 index 00000000000..cf84cbe7db6 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMVAppendFunctionIT.java @@ -0,0 +1,221 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.util.MatcherUtils.*; + +import java.io.IOException; +import java.util.List; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +public class CalciteMVAppendFunctionIT extends PPLIntegTestCase { + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + loadIndex(Index.BANK); + } + + @Test + public void testMvappendWithMultipleElements() throws IOException { + JSONObject actual = + executeQuery( + source(TEST_INDEX_BANK, "eval result = mvappend(1, 2, 3) | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(1, 2, 3))); + } + + @Test + public void testMvappendWithSingleElement() throws IOException { + JSONObject actual = + executeQuery( + source(TEST_INDEX_BANK, "eval result = mvappend(42) | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(42))); + } + + @Test + public void testMvappendWithArrayFlattening() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval arr1 = array(1, 2), arr2 = array(3, 4), result = mvappend(arr1, arr2) | head" + + " 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(1, 2, 3, 4))); + } + + @Test + public void testMvappendWithMixedArrayAndScalar() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval arr = array(1, 2), result = mvappend(arr, 3, 4) | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(1, 2, 3, 4))); + } + + @Test + public void testMvappendWithStringValues() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval result = mvappend('hello', 'world') | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of("hello", "world"))); + } + + @Test + public void testMvappendWithMixedTypes() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval result = mvappend(1, 'text', 2.5) | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(1, "text", 2.5))); + } + + @Test + public void testMvappendWithIntAndDouble() throws IOException { + JSONObject actual = + executeQuery( + source(TEST_INDEX_BANK, "eval result = mvappend(1, 2.5) | head 1 | fields result")); + + System.out.println(actual); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(1, 2.5))); + } + + @Test + public void testMvappendWithRealFields() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval result = mvappend(firstname, lastname) | head 1 | fields firstname, lastname," + + " result")); + + verifySchema( + actual, + schema("firstname", "string"), + schema("lastname", "string"), + schema("result", "array")); + + verifyDataRows( + actual, + rows("Amber JOHnny", "Duke Willmington", List.of("Amber JOHnny", "Duke Willmington"))); + } + + @Test + public void testMvappendWithFieldsAndLiterals() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval result = mvappend(age, 'years', 'old') | head 1 | fields age, result")); + + verifySchema(actual, schema("age", "int"), schema("result", "array")); + verifyDataRows(actual, rows(32, List.of(32, "years", "old"))); + } + + @Test + public void testMvappendWithEmptyArray() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval empty_arr = array(), result = mvappend(empty_arr, 1, 2) | head 1 | fields" + + " result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(1, 2))); + } + + @Test + public void testMvappendWithNestedArrays() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval arr1 = array('a', 'b'), arr2 = array('c'), arr3 = array('d', 'e'), result =" + + " mvappend(arr1, arr2, arr3) | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of("a", "b", "c", "d", "e"))); + } + + @Test + public void testMvappendWithNumericArrays() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval arr1 = array(1.5, 2.5), arr2 = array(3.5), result = mvappend(arr1, arr2, 4.5)" + + " | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(1.5, 2.5, 3.5, 4.5))); + } + + @Test + public void testMvappendInWhereClause() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval combined = mvappend(firstname, lastname) | where array_length(combined) = 2 |" + + " head 1 | fields firstname, lastname, combined")); + + verifySchema( + actual, + schema("firstname", "string"), + schema("lastname", "string"), + schema("combined", "array")); + + verifyDataRows( + actual, + rows("Amber JOHnny", "Duke Willmington", List.of("Amber JOHnny", "Duke Willmington"))); + } + + @Test + public void testMvappendWithComplexExpression() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval result = mvappend(array(age), array(age * 2), age + 10) | head 1 | fields" + + " age, result")); + + verifySchema(actual, schema("age", "int"), schema("result", "array")); + verifyDataRows(actual, rows(32, List.of(32, 64, 42))); + } + + @Test + public void testMvappendWithNull() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval result = mvappend('test', nullif(1, 1), 2) | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of("test", 2))); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultisearchCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultisearchCommandIT.java new file mode 100644 index 00000000000..393b0a4a501 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultisearchCommandIT.java @@ -0,0 +1,367 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.legacy.TestsConstants.*; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +public class CalciteMultisearchCommandIT extends PPLIntegTestCase { + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + loadIndex(Index.ACCOUNT); + loadIndex(Index.BANK); + loadIndex(Index.TIME_TEST_DATA); + loadIndex(Index.TIME_TEST_DATA2); + loadIndex(Index.LOCATIONS_TYPE_CONFLICT); + } + + @Test + public void testBasicMultisearch() throws IOException { + JSONObject result = + executeQuery( + String.format( + "| multisearch " + + "[search source=%s | where age < 30 | eval age_group = \\\"young\\\"] " + + "[search source=%s | where age >= 30 | eval age_group = \\\"adult\\\"] " + + "| stats count by age_group | sort age_group", + TEST_INDEX_ACCOUNT, TEST_INDEX_ACCOUNT)); + + verifySchema(result, schema("count", null, "bigint"), schema("age_group", null, "string")); + verifyDataRows(result, rows(549L, "adult"), rows(451L, "young")); + } + + @Test + public void testMultisearchSuccessRatePattern() throws IOException { + JSONObject result = + executeQuery( + String.format( + "| multisearch " + + "[search source=%s | where balance > 20000 | eval query_type = \\\"good\\\"] " + + "[search source=%s | where balance > 0 | eval query_type = \\\"valid\\\"] " + + "| stats count(eval(query_type = \\\"good\\\")) as good_accounts, " + + " count(eval(query_type = \\\"valid\\\")) as total_valid", + TEST_INDEX_ACCOUNT, TEST_INDEX_ACCOUNT)); + + verifySchema( + result, schema("good_accounts", null, "bigint"), schema("total_valid", null, "bigint")); + + verifyDataRows(result, rows(619L, 1000L)); + } + + @Test + public void testMultisearchWithThreeSubsearches() throws IOException { + JSONObject result = + executeQuery( + String.format( + "| multisearch [search source=%s | where state = \\\"IL\\\" | eval region" + + " = \\\"Illinois\\\"] [search source=%s | where state = \\\"TN\\\" | eval" + + " region = \\\"Tennessee\\\"] [search source=%s | where state = \\\"CA\\\" |" + + " eval region = \\\"California\\\"] | stats count by region | sort region", + TEST_INDEX_ACCOUNT, TEST_INDEX_ACCOUNT, TEST_INDEX_ACCOUNT)); + + verifySchema(result, schema("count", null, "bigint"), schema("region", null, "string")); + + verifyDataRows(result, rows(17L, "California"), rows(22L, "Illinois"), rows(25L, "Tennessee")); + } + + @Test + public void testMultisearchWithComplexAggregation() throws IOException { + JSONObject result = + executeQuery( + String.format( + "| multisearch [search source=%s | where gender = \\\"M\\\" | eval" + + " segment = \\\"male\\\"] [search source=%s | where gender = \\\"F\\\" | eval" + + " segment = \\\"female\\\"] | stats count as customer_count, avg(balance) as" + + " avg_balance by segment | sort segment", + TEST_INDEX_ACCOUNT, TEST_INDEX_ACCOUNT)); + + verifySchema( + result, + schema("customer_count", null, "bigint"), + schema("avg_balance", null, "double"), + schema("segment", null, "string")); + + verifyDataRows( + result, rows(493L, 25623.34685598377, "female"), rows(507L, 25803.800788954635, "male")); + } + + @Test + public void testMultisearchWithEmptySubsearch() throws IOException { + JSONObject result = + executeQuery( + String.format( + "| multisearch " + + "[search source=%s | where age > 25] " + + "[search source=%s | where age > 200 | eval impossible = \\\"yes\\\"] " + + "| stats count", + TEST_INDEX_ACCOUNT, TEST_INDEX_ACCOUNT)); + + verifySchema(result, schema("count", null, "bigint")); + + verifyDataRows(result, rows(733L)); + } + + @Test + public void testMultisearchWithFieldsProjection() throws IOException { + JSONObject result = + executeQuery( + String.format( + "| multisearch [search source=%s | where gender = \\\"M\\\" | fields" + + " firstname, lastname, balance] [search source=%s | where gender = \\\"F\\\"" + + " | fields firstname, lastname, balance] | head 5", + TEST_INDEX_ACCOUNT, TEST_INDEX_ACCOUNT)); + + verifySchema( + result, + schema("firstname", null, "string"), + schema("lastname", null, "string"), + schema("balance", null, "bigint")); + + verifyDataRows( + result, + rows("Amber", "Duke", 39225L), + rows("Hattie", "Bond", 5686L), + rows("Dale", "Adams", 4180L), + rows("Elinor", "Ratliff", 16418L), + rows("Mcgee", "Mooney", 18612L)); + } + + @Test + public void testMultisearchWithTimestampInterleaving() throws IOException { + JSONObject result = + executeQuery( + "| multisearch [search" + + " source=opensearch-sql_test_index_time_data | where category IN (\\\"A\\\"," + + " \\\"B\\\")] [search source=opensearch-sql_test_index_time_data2 | where" + + " category IN (\\\"E\\\", \\\"F\\\")] | head 10"); + + verifySchema( + result, + schema("@timestamp", null, "string"), + schema("category", null, "string"), + schema("value", null, "int"), + schema("timestamp", null, "string")); + + verifyDataRows( + result, + rows("2025-08-01 04:00:00", "E", 2001, "2025-08-01 04:00:00"), + rows("2025-08-01 03:47:41", "A", 8762, "2025-08-01 03:47:41"), + rows("2025-08-01 02:30:00", "F", 2002, "2025-08-01 02:30:00"), + rows("2025-08-01 01:14:11", "B", 9015, "2025-08-01 01:14:11"), + rows("2025-08-01 01:00:00", "E", 2003, "2025-08-01 01:00:00"), + rows("2025-07-31 23:40:33", "A", 8676, "2025-07-31 23:40:33"), + rows("2025-07-31 22:15:00", "F", 2004, "2025-07-31 22:15:00"), + rows("2025-07-31 21:07:03", "B", 8490, "2025-07-31 21:07:03"), + rows("2025-07-31 20:45:00", "E", 2005, "2025-07-31 20:45:00"), + rows("2025-07-31 19:33:25", "A", 9231, "2025-07-31 19:33:25")); + } + + @Test + public void testMultisearchWithNonStreamingCommands() throws IOException { + JSONObject result = + executeQuery( + String.format( + "| multisearch " + + "[search source=%s | where age < 30 | stats count() as young_count] " + + "[search source=%s | where age >= 30 | stats count() as adult_count] " + + "| stats sum(young_count) as total_young, sum(adult_count) as total_adult", + TEST_INDEX_ACCOUNT, TEST_INDEX_ACCOUNT)); + + verifySchema( + result, schema("total_young", null, "bigint"), schema("total_adult", null, "bigint")); + + verifyDataRows(result, rows(451L, 549L)); + } + + @Test + public void testMultisearchWithSingleSubsearchThrowsError() { + Exception exception = + assertThrows( + ResponseException.class, + () -> + executeQuery( + String.format( + "| multisearch " + "[search source=%s | where age > 30]", + TEST_INDEX_ACCOUNT))); + + assertTrue( + "Error message should indicate minimum subsearch requirement", + exception.getMessage().contains("Multisearch command requires at least two subsearches")); + } + + @Test + public void testMultisearchWithDifferentIndicesSchemaMerge() throws IOException { + JSONObject result = + executeQuery( + String.format( + "| multisearch [search source=%s | where age > 35 | fields account_number," + + " firstname, balance] [search source=%s | where age > 35 | fields" + + " account_number, balance] | stats count() as total_count", + TEST_INDEX_ACCOUNT, TEST_INDEX_BANK)); + + verifySchema(result, schema("total_count", null, "bigint")); + verifyDataRows(result, rows(241L)); + } + + @Test + public void testMultisearchWithMixedIndicesComplexSchemaMerge() throws IOException { + JSONObject result = + executeQuery( + String.format( + "| multisearch [search source=%s | where balance > 40000 | eval record_type =" + + " \\\"financial\\\" | fields account_number, balance, record_type] [search" + + " source=%s | where value > 5000 | eval record_type = \\\"timeseries\\\" |" + + " fields value, category, record_type] | stats count by record_type | sort" + + " record_type", + TEST_INDEX_ACCOUNT, "opensearch-sql_test_index_time_data")); + + verifySchema(result, schema("count", null, "bigint"), schema("record_type", null, "string")); + verifyDataRows(result, rows(215L, "financial"), rows(100L, "timeseries")); + } + + @Test + public void testMultisearchWithTimeIndicesTimestampOrdering() throws IOException { + JSONObject result = + executeQuery( + String.format( + "| multisearch " + + "[search source=%s | where category = \\\"A\\\" | stats count() as count_a] " + + "[search source=%s | where category = \\\"E\\\" | stats count() as count_e] " + + "| stats sum(count_a) as total_a, sum(count_e) as total_e", + "opensearch-sql_test_index_time_data", "opensearch-sql_test_index_time_data2")); + + verifySchema(result, schema("total_a", null, "bigint"), schema("total_e", null, "bigint")); + verifyDataRows(result, rows(26L, 10L)); + } + + @Test + public void testMultisearchNullFillingForMissingFields() throws IOException { + JSONObject result = + executeQuery( + String.format( + "| multisearch [search source=%s | where account_number = 1 | fields firstname," + + " age, balance] [search source=%s | where account_number = 1 | fields" + + " lastname, city, employer] | head 2", + TEST_INDEX_ACCOUNT, TEST_INDEX_ACCOUNT)); + + verifySchema( + result, + schema("firstname", null, "string"), + schema("age", null, "bigint"), + schema("balance", null, "bigint"), + schema("lastname", null, "string"), + schema("city", null, "string"), + schema("employer", null, "string")); + + verifyDataRows( + result, + rows("Amber", 32L, 39225L, null, null, null), + rows(null, null, null, "Duke", "Brogan", "Pyrami")); + } + + @Test + public void testMultisearchNullFillingAcrossIndices() throws IOException { + JSONObject result = + executeQuery( + String.format( + "| multisearch [search source=%s | where account_number = 1 | fields" + + " account_number, firstname, balance] [search source=%s | where" + + " account_number = 1 | fields city, employer, email] | head 2", + TEST_INDEX_ACCOUNT, TEST_INDEX_BANK)); + + verifySchema( + result, + schema("account_number", null, "bigint"), + schema("firstname", null, "string"), + schema("balance", null, "bigint"), + schema("city", null, "string"), + schema("employer", null, "string"), + schema("email", null, "string")); + + verifyDataRows( + result, + rows(1L, "Amber", 39225L, null, null, null), + rows(null, null, null, "Brogan", "Pyrami", "amberduke@pyrami.com")); + } + + @Test + public void testMultisearchWithDirectTypeConflict() { + Exception exception = + assertThrows( + ResponseException.class, + () -> + executeQuery( + String.format( + "| multisearch " + + "[search source=%s | fields firstname, age, balance | head 2] " + + "[search source=%s | fields description, age, place_id | head 2]", + TEST_INDEX_ACCOUNT, TEST_INDEX_LOCATIONS_TYPE_CONFLICT))); + + assertTrue( + "Error message should indicate type conflict", + exception + .getMessage() + .contains("Unable to process column 'age' due to incompatible types:")); + } + + @Test + public void testMultisearchCrossIndexFieldSelection() throws IOException { + JSONObject result = + executeQuery( + String.format( + "| multisearch " + + "[search source=%s | fields firstname, balance | head 2] " + + "[search source=%s | fields description, place_id | head 2]", + TEST_INDEX_ACCOUNT, TEST_INDEX_LOCATIONS_TYPE_CONFLICT)); + + verifySchema( + result, + schema("firstname", null, "string"), + schema("balance", null, "bigint"), + schema("description", null, "string"), + schema("place_id", null, "int")); + + verifyDataRows( + result, + rows("Amber", 39225L, null, null), + rows("Hattie", 5686L, null, null), + rows(null, null, "Central Park", 1001), + rows(null, null, "Times Square", 1002)); + } + + @Test + public void testMultisearchTypeConflictWithStats() { + Exception exception = + assertThrows( + ResponseException.class, + () -> + executeQuery( + String.format( + "| multisearch " + + "[search source=%s | fields age] " + + "[search source=%s | fields age] " + + "| stats count() as total", + TEST_INDEX_ACCOUNT, TEST_INDEX_LOCATIONS_TYPE_CONFLICT))); + + assertTrue( + "Error message should indicate type conflict", + exception + .getMessage() + .contains("Unable to process column 'age' due to incompatible types:")); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java index c280c5cb8b2..e0872dc543c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java @@ -16,6 +16,7 @@ import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifyErrorMessageContains; import static org.opensearch.sql.util.MatcherUtils.verifySchema; import static org.opensearch.sql.util.MatcherUtils.verifySchemaInOrder; @@ -25,6 +26,8 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.client.Request; +import org.opensearch.sql.common.utils.StringUtils; +import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.ppl.PPLIntegTestCase; public class CalcitePPLAggregationIT extends PPLIntegTestCase { @@ -41,6 +44,7 @@ public void init() throws Exception { loadIndex(Index.CALCS); loadIndex(Index.DATE_FORMATS); loadIndex(Index.DATA_TYPE_NUMERIC); + loadIndex(Index.BIG5); loadIndex(Index.LOGS); loadIndex(Index.TELEMETRY); loadIndex(Index.TIME_TEST_DATA); @@ -729,6 +733,23 @@ public void testCountBySpanForCustomFormats() throws IOException { verifyDataRows(actual, rows(1, "00:00:00"), rows(1, "12:00:00")); } + // Only available in v3 with Calcite + @Test + public void testSpanByImplicitTimestamp() throws IOException { + JSONObject result = executeQuery("source=big5 | stats count() by span(1d) as span"); + verifySchema(result, schema("count()", "bigint"), schema("span", "timestamp")); + verifyDataRows(result, rows(1, "2023-01-02 00:00:00")); + + Throwable t = + assertThrowsWithReplace( + SemanticCheckException.class, + () -> + executeQuery( + StringUtils.format( + "source=%s | stats count() by span(5m)", TEST_INDEX_DATE_FORMATS))); + verifyErrorMessageContains(t, "Field [@timestamp] not found"); + } + @Test public void testCountDistinct() throws IOException { JSONObject actual = diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendCommandIT.java index d971a6f3cb1..d01ddfb2a44 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendCommandIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOGS; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -18,6 +19,7 @@ import java.util.Locale; import org.json.JSONObject; import org.junit.Test; +import org.opensearch.client.ResponseException; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.ppl.PPLIntegTestCase; @@ -28,6 +30,7 @@ public void init() throws Exception { enableCalcite(); loadIndex(Index.ACCOUNT); loadIndex(Index.BANK); + loadIndex(Index.WEBLOG); } @Test @@ -213,27 +216,59 @@ public void testAppendWithMergedColumn() throws IOException { } @Test - public void testAppendWithConflictTypeColumn() throws IOException { + public void testAppendWithConflictTypeColumn() { + Exception exception = + assertThrows( + ResponseException.class, + () -> + executeQuery( + String.format( + Locale.ROOT, + "source=%s | stats sum(age) as sum by gender | append [ source=%s | stats" + + " sum(age) as sum by state | sort sum | eval sum = cast(sum as" + + " double) ] | head 5", + TEST_INDEX_ACCOUNT, + TEST_INDEX_ACCOUNT))); + + assertTrue( + "Error message should indicate type conflict", + exception + .getMessage() + .contains("Unable to process column 'sum' due to incompatible types:")); + } + + @Test + public void testAppendSchemaMergeWithTimestampUDT() throws IOException { JSONObject actual = executeQuery( String.format( Locale.ROOT, - "source=%s | stats sum(age) as sum by gender | append [ source=%s | stats sum(age)" - + " as sum by state | sort sum | eval sum = cast(sum as double) ] | head 5", + "source=%s | fields account_number, firstname | append [ source=%s | fields" + + " account_number, age, birthdate ] | where isnotnull(birthdate) and" + + " account_number > 30", TEST_INDEX_ACCOUNT, - TEST_INDEX_ACCOUNT)); + TEST_INDEX_BANK)); verifySchemaInOrder( actual, - schema("sum", "bigint"), - schema("gender", "string"), - schema("state", "string"), - schema("sum0", "double")); - verifyDataRows( - actual, - rows(14947, "F", null, null), - rows(15224, "M", null, null), - rows(null, null, "NV", 369d), - rows(null, null, "NM", 412d), - rows(null, null, "AZ", 414d)); + schema("account_number", "bigint"), + schema("firstname", "string"), + schema("age", "int"), + schema("birthdate", "string")); + verifyDataRows(actual, rows(32, null, 34, "2018-08-11 00:00:00")); + } + + @Test + public void testAppendSchemaMergeWithIpUDT() throws IOException { + JSONObject actual = + executeQuery( + String.format( + Locale.ROOT, + "source=%s | fields account_number, age | append [ source=%s | fields host ] |" + + " where cidrmatch(host, '0.0.0.0/24')", + TEST_INDEX_ACCOUNT, + TEST_INDEX_WEBLOGS)); + verifySchemaInOrder( + actual, schema("account_number", "bigint"), schema("age", "bigint"), schema("host", "ip")); + verifyDataRows(actual, rows(null, null, "0.0.0.2")); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java index 21e02f77afe..5f69159fec5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java @@ -95,10 +95,7 @@ public void testFieldsShouldBeCaseSensitive() { Throwable e = assertThrowsWithReplace( IllegalStateException.class, () -> executeQuery("source=test | fields NAME")); - verifyErrorMessageContains( - e, - "field [NAME] not found; input fields are: [name, age, _id, _index, _score, _maxscore," - + " _sort, _routing]"); + verifyErrorMessageContains(e, "Field [NAME] not found."); } @Test @@ -537,11 +534,7 @@ public void testKeepThrowCalciteException() throws IOException { () -> executeQuery( String.format("source=%s | fields firstname1, age", TEST_INDEX_BANK))); - verifyErrorMessageContains( - e, - "field [firstname1] not found; input fields are: [account_number, firstname, address," - + " birthdate, gender, city, lastname, balance, employer, state, age, email," - + " male, _id, _index, _score, _maxscore, _sort, _routing]"); + verifyErrorMessageContains(e, "Field [firstname1] not found."); }, ""); } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java index 7e4425d3a41..b7e16d1da8b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java @@ -5,14 +5,20 @@ package org.opensearch.sql.calcite.remote; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_OTEL_LOGS; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY_WITH_NULL; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOGS; +import static org.opensearch.sql.util.MatcherUtils.closeTo; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifyNumOfRows; import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; import org.json.JSONObject; +import org.junit.Assume; import org.junit.jupiter.api.Test; import org.opensearch.client.Request; import org.opensearch.sql.legacy.TestsConstants; @@ -25,6 +31,10 @@ public void init() throws Exception { enableCalcite(); loadIndex(Index.WEBLOG); + loadIndex(Index.TIME_TEST_DATA); + loadIndex(Index.STATE_COUNTRY_WITH_NULL); + loadIndex(Index.BANK); + loadIndex(Index.OTELLOGS); appendDataForBadResponse(); } @@ -246,4 +256,269 @@ public void testCaseWhenInSubquery() throws IOException { rows("0.0.0.2", "GET", null, "4085", "500", "/shuttle/missions/sts-73/mission-sts-73.html"), rows("::3", "GET", null, "3985", "403", "/shuttle/countdown/countdown.html")); } + + @Test + public void testCaseCanBePushedDownAsRangeQuery() throws IOException { + // CASE 1: Range - Metric + // 1.1 Range - Metric + JSONObject actual1 = + executeQuery( + String.format( + "source=%s | eval age_range = case(age < 30, 'u30', age < 40, 'u40' else 'u100') |" + + " stats avg(age) as avg_age by age_range", + TEST_INDEX_BANK)); + verifySchema(actual1, schema("avg_age", "double"), schema("age_range", "string")); + verifyDataRows(actual1, rows(28.0, "u30"), rows(35.0, "u40")); + + // 1.2 Range - Metric (COUNT) + JSONObject actual2 = + executeQuery( + String.format( + "source=%s | eval age_range = case(age < 30, 'u30', age >= 30 and age < 40, 'u40'" + + " else 'u100') | stats avg(age) by age_range", + TEST_INDEX_BANK)); + verifySchema(actual2, schema("avg(age)", "double"), schema("age_range", "string")); + verifyDataRows(actual2, rows(28.0, "u30"), rows(35.0, "u40")); + + // 1.3 Range - Range - Metric + JSONObject actual3 = + executeQuery( + String.format( + "source=%s | eval age_range = case(age < 30, 'u30', age < 40, 'u40' else 'u100')," + + " balance_range = case(balance < 20000, 'medium' else 'high') | stats" + + " avg(balance) as avg_balance by age_range, balance_range", + TEST_INDEX_BANK)); + verifySchema( + actual3, + schema("avg_balance", "double"), + schema("age_range", "string"), + schema("balance_range", "string")); + verifyDataRows( + actual3, + rows(32838.0, "u30", "high"), + closeTo(8761.333333333334, "u40", "medium"), + rows(42617.0, "u40", "high")); + + // 1.4 Range - Metric (With null & discontinuous ranges) + JSONObject actual4 = + executeQuery( + String.format( + "source=%s | eval age_range = case(age < 30, 'u30', (age >= 35 and age < 40) or age" + + " >= 80, '30-40 or >=80') | stats avg(balance) by age_range", + TEST_INDEX_BANK)); + verifySchema(actual4, schema("avg(balance)", "double"), schema("age_range", "string")); + // There's such a discrepancy because null cannot be the key for a range query + if (isPushdownDisabled()) { + verifyDataRows( + actual4, + rows(32838.0, "u30"), + rows(30497.0, null), + closeTo(20881.333333333332, "30-40 or >=80")); + } else { + verifyDataRows( + actual4, + rows(32838.0, "u30"), + rows(30497.0, "null"), + closeTo(20881.333333333332, "30-40 or >=80")); + } + + // 1.5 Should not be pushed because the range is not closed-open + JSONObject actual5 = + executeQuery( + String.format( + "source=%s | eval age_range = case(age < 30, 'u30', age >= 30 and age <= 40, 'u40'" + + " else 'u100') | stats avg(age) as avg_age by age_range", + TEST_INDEX_BANK)); + verifySchema(actual5, schema("avg_age", "double"), schema("age_range", "string")); + verifyDataRows(actual5, rows(35.0, "u40"), rows(28.0, "u30")); + } + + @Test + public void testCaseCanBePushedDownAsCompositeRangeQuery() throws IOException { + // CASE 2: Composite - Range - Metric + // 2.1 Composite (term) - Range - Metric + JSONObject actual6 = + executeQuery( + String.format( + "source=%s | eval age_range = case(age < 30, 'u30' else 'a30') | stats avg(balance)" + + " by state, age_range", + TEST_INDEX_BANK)); + verifySchema( + actual6, + schema("avg(balance)", "double"), + schema("state", "string"), + schema("age_range", "string")); + verifyDataRows( + actual6, + rows(39225.0, "IL", "a30"), + rows(48086.0, "IN", "a30"), + rows(4180.0, "MD", "a30"), + rows(40540.0, "PA", "a30"), + rows(5686.0, "TN", "a30"), + rows(32838.0, "VA", "u30"), + rows(16418.0, "WA", "a30")); + + // 2.2 Composite (date histogram) - Range - Metric + JSONObject actual7 = + executeQuery( + "source=opensearch-sql_test_index_time_data | eval value_range = case(value < 7000," + + " 'small' else 'large') | stats avg(value) by value_range, span(@timestamp," + + " 1month)"); + verifySchema( + actual7, + schema("avg(value)", "double"), + schema("span(@timestamp,1month)", "timestamp"), + schema("value_range", "string")); + + verifyDataRows( + actual7, + closeTo(6642.521739130435, "2025-07-01 00:00:00", "small"), + closeTo(8381.917808219177, "2025-07-01 00:00:00", "large"), + rows(6489.0, "2025-08-01 00:00:00", "small"), + rows(8375.0, "2025-08-01 00:00:00", "large")); + + // 2.3 Composite(2 fields) - Range - Metric (with count) + JSONObject actual8 = + executeQuery( + String.format( + "source=%s | eval age_range = case(age < 30, 'u30' else 'a30') | stats" + + " avg(balance), count() by age_range, state, gender", + TEST_INDEX_BANK)); + verifySchema( + actual8, + schema("avg(balance)", "double"), + schema("count()", "bigint"), + schema("age_range", "string"), + schema("state", "string"), + schema("gender", "string")); + verifyDataRows( + actual8, + rows(5686.0, 1, "a30", "TN", "M"), + rows(16418.0, 1, "a30", "WA", "M"), + rows(40540.0, 1, "a30", "PA", "F"), + rows(4180.0, 1, "a30", "MD", "M"), + rows(32838.0, 1, "u30", "VA", "F"), + rows(39225.0, 1, "a30", "IL", "M"), + rows(48086.0, 1, "a30", "IN", "F")); + + // 2.4 Composite (2 fields) - Range - Range - Metric (with count) + JSONObject actual9 = + executeQuery( + String.format( + "source=%s | eval age_range = case(age < 35, 'u35' else 'a35'), balance_range =" + + " case(balance < 20000, 'medium' else 'high') | stats avg(balance) as" + + " avg_balance by age_range, balance_range, state", + TEST_INDEX_BANK)); + verifySchema( + actual9, + schema("avg_balance", "double"), + schema("age_range", "string"), + schema("balance_range", "string"), + schema("state", "string")); + verifyDataRows( + actual9, + rows(39225.0, "u35", "high", "IL"), + rows(48086.0, "u35", "high", "IN"), + rows(4180.0, "u35", "medium", "MD"), + rows(40540.0, "a35", "high", "PA"), + rows(5686.0, "a35", "medium", "TN"), + rows(32838.0, "u35", "high", "VA"), + rows(16418.0, "a35", "medium", "WA")); + + // 2.5 Should not be pushed because case result expression is not constant + JSONObject actual10 = + executeQuery( + String.format( + "source=%s | eval age_range = case(age < 35, 'u35' else email) | stats avg(balance)" + + " as avg_balance by age_range, state", + TEST_INDEX_BANK)); + verifySchema( + actual10, + schema("avg_balance", "double"), + schema("age_range", "string"), + schema("state", "string")); + verifyDataRows( + actual10, + rows(32838.0, "u35", "VA"), + rows(4180.0, "u35", "MD"), + rows(48086.0, "u35", "IN"), + rows(40540.0, "virginiaayala@filodyne.com", "PA"), + rows(39225.0, "u35", "IL"), + rows(5686.0, "hattiebond@netagy.com", "TN"), + rows(16418.0, "elinorratliff@scentric.com", "WA")); + } + + @Test + public void testCaseAggWithNullValues() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s" + + "| eval age_category = case(" + + " age < 20, 'teenager'," + + " age < 70, 'adult'," + + " age >= 70, 'senior'" + + " else 'unknown')" + + "| stats avg(age) by age_category", + TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + verifySchema(actual, schema("avg(age)", "double"), schema("age_category", "string")); + // There is such discrepancy because range aggregations will ignore null values + if (isPushdownDisabled()) { + verifyDataRows( + actual, + rows(10, "teenager"), + rows(25, "adult"), + rows(70, "senior"), + rows(null, "unknown")); + } else { + verifyDataRows(actual, rows(10, "teenager"), rows(25, "adult"), rows(70, "senior")); + } + } + + @Test + public void testNestedCaseAggWithAutoDateHistogram() throws IOException { + // TODO: Remove after resolving: https://github.com/opensearch-project/sql/issues/4578 + Assume.assumeFalse( + "The query cannot be executed when pushdown is disabled due to implementation defects of" + + " the bin command", + isPushdownDisabled()); + JSONObject actual1 = + executeQuery( + String.format( + "source=%s | bin @timestamp bins=2 | eval severity_range = case(severityNumber <" + + " 16, 'minor' else 'severe') | stats avg(severityNumber), count() by" + + " @timestamp, severity_range, flags", + TEST_INDEX_OTEL_LOGS)); + verifySchema( + actual1, + schema("avg(severityNumber)", "double"), + schema("count()", "bigint"), + schema("@timestamp", "timestamp"), + schema("severity_range", "string"), + schema("flags", "bigint")); + + verifyDataRows( + actual1, + rows(8.85, 20, "2024-01-15 10:30:02", "minor", 0), + rows(20, 9, "2024-01-15 10:30:02", "severe", 0), + rows(9, 1, "2024-01-15 10:30:00", "minor", 1), + rows(17, 1, "2024-01-15 10:30:00", "severe", 1), + rows(1, 1, "2024-01-15 10:30:05", "minor", 1)); + + JSONObject actual2 = + executeQuery( + String.format( + "source=%s | bin @timestamp bins=100 | eval severity_range = case(severityNumber <" + + " 16, 'minor' else 'severe') | stats avg(severityNumber), count() by" + + " @timestamp, severity_range, flags", + TEST_INDEX_OTEL_LOGS)); + verifySchema( + actual2, + schema("avg(severityNumber)", "double"), + schema("count()", "bigint"), + schema("@timestamp", "timestamp"), + schema("severity_range", "string"), + schema("flags", "bigint")); + verifyNumOfRows(actual2, 32); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLConditionBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLConditionBuiltinFunctionIT.java index af2cf5e964d..f7c81e797df 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLConditionBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLConditionBuiltinFunctionIT.java @@ -335,4 +335,71 @@ public void testEarliestWithEval() throws IOException { verifyDataRows(actual, rows(false, true)); } + + @Test + public void testEvalIsNullWithIf() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval n=if(isnull(name), 'yes', 'no') | fields name, n", + TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + + verifySchema(actual, schema("name", "string"), schema("n", "string")); + + verifyDataRows( + actual, + rows("John", "no"), + rows("Jane", "no"), + rows(null, "yes"), + rows("Jake", "no"), + rows("Kevin", "no"), + rows("Hello", "no"), + rows(" ", "no"), + rows("", "no")); + } + + @Test + public void testEvalIsNotNullDirect() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval is_not_null_name=isnotnull(name) | fields name, is_not_null_name", + TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + + verifySchema(actual, schema("name", "string"), schema("is_not_null_name", "boolean")); + + verifyDataRows( + actual, + rows("John", true), + rows("Jane", true), + rows(null, false), + rows("Jake", true), + rows("Kevin", true), + rows("Hello", true), + rows(" ", true), + rows("", true)); + } + + @Test + public void testEvalIsNullInComplexExpression() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval safe_name=if(isnull(name), 'Unknown', name) | fields safe_name," + + " age", + TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + + verifySchema(actual, schema("safe_name", "string"), schema("age", "int")); + + verifyDataRows( + actual, + rows("John", 25), + rows("Jane", 20), + rows("Unknown", 10), + rows("Jake", 70), + rows("Kevin", null), + rows("Hello", 30), + rows(" ", 27), + rows("", 57)); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLEventstatsIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLEventstatsIT.java index 59c23a0eeed..9839fff00c4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLEventstatsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLEventstatsIT.java @@ -27,7 +27,7 @@ public void init() throws Exception { } @Test - public void testEventstat() throws IOException { + public void testEventstats() throws IOException { JSONObject actual = executeQuery( String.format( @@ -57,7 +57,7 @@ public void testEventstat() throws IOException { } @Test - public void testEventstatWithNull() throws IOException { + public void testEventstatsWithNull() throws IOException { JSONObject actual = executeQuery( String.format( @@ -89,7 +89,7 @@ public void testEventstatWithNull() throws IOException { } @Test - public void testEventstatBy() throws IOException { + public void testEventstatsBy() throws IOException { JSONObject actual = executeQuery( String.format( @@ -119,7 +119,7 @@ public void testEventstatBy() throws IOException { } @Test - public void testEventstatByWithNull() throws IOException { + public void testEventstatsByWithNull() throws IOException { JSONObject actual = executeQuery( String.format( @@ -166,7 +166,7 @@ public void testEventstatByWithNull() throws IOException { } @Test - public void testEventstatBySpan() throws IOException { + public void testEventstatsBySpan() throws IOException { JSONObject actual = executeQuery( String.format( @@ -183,7 +183,7 @@ public void testEventstatBySpan() throws IOException { } @Test - public void testEventstatBySpanWithNull() throws IOException { + public void testEventstatsBySpanWithNull() throws IOException { JSONObject actual = executeQuery( String.format( @@ -202,7 +202,7 @@ public void testEventstatBySpanWithNull() throws IOException { } @Test - public void testEventstatByMultiplePartitions1() throws IOException { + public void testEventstatsByMultiplePartitions1() throws IOException { JSONObject actual = executeQuery( String.format( @@ -219,7 +219,7 @@ public void testEventstatByMultiplePartitions1() throws IOException { } @Test - public void testEventstatByMultiplePartitions2() throws IOException { + public void testEventstatsByMultiplePartitions2() throws IOException { JSONObject actual = executeQuery( String.format( @@ -236,7 +236,7 @@ public void testEventstatByMultiplePartitions2() throws IOException { } @Test - public void testEventstatByMultiplePartitionsWithNull1() throws IOException { + public void testEventstatsByMultiplePartitionsWithNull1() throws IOException { JSONObject actual = executeQuery( String.format( @@ -255,7 +255,7 @@ public void testEventstatByMultiplePartitionsWithNull1() throws IOException { } @Test - public void testEventstatByMultiplePartitionsWithNull2() throws IOException { + public void testEventstatsByMultiplePartitionsWithNull2() throws IOException { JSONObject actual = executeQuery( String.format( @@ -289,7 +289,7 @@ public void testUnsupportedWindowFunctions() { } @Test - public void testMultipleEventstat() throws IOException { + public void testMultipleEventstats() throws IOException { JSONObject actual = executeQuery( String.format( @@ -306,7 +306,7 @@ public void testMultipleEventstat() throws IOException { } @Test - public void testMultipleEventstatWithNull() throws IOException { + public void testMultipleEventstatsWithNull() throws IOException { JSONObject actual = executeQuery( String.format( @@ -325,7 +325,7 @@ public void testMultipleEventstatWithNull() throws IOException { } @Test - public void testMultipleEventstatWithEval() throws IOException { + public void testMultipleEventstatsWithEval() throws IOException { JSONObject actual = executeQuery( String.format( @@ -343,7 +343,7 @@ public void testMultipleEventstatWithEval() throws IOException { } @Test - public void testEventstatEmptyRows() throws IOException { + public void testEventstatsEmptyRows() throws IOException { JSONObject actual = executeQuery( String.format( @@ -363,7 +363,7 @@ public void testEventstatEmptyRows() throws IOException { } @Test - public void testEventstatVariance() throws IOException { + public void testEventstatsVariance() throws IOException { JSONObject actual = executeQuery( String.format( @@ -433,7 +433,7 @@ public void testEventstatVariance() throws IOException { } @Test - public void testEventstatVarianceWithNull() throws IOException { + public void testEventstatsVarianceWithNull() throws IOException { JSONObject actual = executeQuery( String.format( @@ -496,7 +496,7 @@ public void testEventstatVarianceWithNull() throws IOException { } @Test - public void testEventstatVarianceBy() throws IOException { + public void testEventstatsVarianceBy() throws IOException { JSONObject actual = executeQuery( String.format( @@ -513,7 +513,7 @@ public void testEventstatVarianceBy() throws IOException { } @Test - public void testEventstatVarianceBySpan() throws IOException { + public void testEventstatsVarianceBySpan() throws IOException { JSONObject actual = executeQuery( String.format( @@ -527,7 +527,7 @@ public void testEventstatVarianceBySpan() throws IOException { } @Test - public void testEventstatVarianceWithNullBy() throws IOException { + public void testEventstatsVarianceWithNullBy() throws IOException { JSONObject actual = executeQuery( String.format( @@ -576,7 +576,7 @@ public void testEventstatVarianceWithNullBy() throws IOException { } @Test - public void testEventstatDistinctCount() throws IOException { + public void testEventstatsDistinctCount() throws IOException { JSONObject actual = executeQuery( String.format( @@ -601,7 +601,7 @@ public void testEventstatDistinctCount() throws IOException { } @Test - public void testEventstatDistinctCountByCountry() throws IOException { + public void testEventstatsDistinctCountByCountry() throws IOException { JSONObject actual = executeQuery( String.format( @@ -627,7 +627,7 @@ public void testEventstatDistinctCountByCountry() throws IOException { } @Test - public void testEventstatDistinctCountFunction() throws IOException { + public void testEventstatsDistinctCountFunction() throws IOException { JSONObject actual = executeQuery( String.format( @@ -653,7 +653,7 @@ public void testEventstatDistinctCountFunction() throws IOException { } @Test - public void testEventstatDistinctCountWithNull() throws IOException { + public void testEventstatsDistinctCountWithNull() throws IOException { JSONObject actual = executeQuery( String.format( @@ -681,7 +681,7 @@ public void testEventstatDistinctCountWithNull() throws IOException { } @Test - public void testEventstatEarliestAndLatest() throws IOException { + public void testEventstatsEarliestAndLatest() throws IOException { JSONObject actual = executeQuery( String.format( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExistsSubqueryIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExistsSubqueryIT.java index e21f828b210..73e5c49987b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExistsSubqueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExistsSubqueryIT.java @@ -282,4 +282,160 @@ public void testIssue3566() throws IOException { verifySchemaInOrder(result, schema("count()", "bigint"), schema("country", "string")); verifyDataRows(result, rows(1, null), rows(1, "England"), rows(1, "USA"), rows(2, "Canada")); } + + @Test + public void testSubsearchMaxOut1() throws IOException { + setSubsearchMaxOut(1); + JSONObject result = + executeQuery( + String.format( + "source = %s" + + "| where exists [" + + " source = %s | where id = uid" + + " ]" + + "| sort - salary" + + "| fields id, name, salary", + TEST_INDEX_WORKER, TEST_INDEX_WORK_INFORMATION)); + verifyNumOfRows(result, 1); + resetSubsearchMaxOut(); + } + + @Test + public void testSubsearchMaxOut2() throws IOException { + setSubsearchMaxOut(2); + JSONObject result = + executeQuery( + String.format( + "source = %s" + + "| where exists [" + + " source = %s | where id = uid and department = 'DATA'" + + " ]" + + "| sort - salary" + + "| fields id, name, salary", + TEST_INDEX_WORKER, TEST_INDEX_WORK_INFORMATION)); + verifyNumOfRows(result, 2); + resetSubsearchMaxOut(); + } + + @Test + public void testSubsearchMaxOut3() throws IOException { + setSubsearchMaxOut(2); + JSONObject result = + executeQuery( + String.format( + "source = %s" + + "| where exists [" + + " source = %s " + + " | where id = uid " + + " | eval dept = department " + + " | where dept = 'DATA' " + + " | sort - dept" + + " ]" + + "| sort - salary" + + "| fields id, name, salary", + TEST_INDEX_WORKER, TEST_INDEX_WORK_INFORMATION)); + verifyNumOfRows(result, 1); + resetSubsearchMaxOut(); + } + + @Test + public void testSubsearchMaxOut4() throws IOException { + setSubsearchMaxOut(2); + JSONObject result = + executeQuery( + String.format( + "source = %s" + + "| where exists [" + + " source = %s " + + " | eval dept = department " + + " | where dept = 'DATA' " + + " | where id = uid" + + " ]" + + "| sort - salary" + + "| fields id, name, salary", + TEST_INDEX_WORKER, TEST_INDEX_WORK_INFORMATION)); + verifyNumOfRows(result, 2); + resetSubsearchMaxOut(); + } + + @Test + public void testSubsearchMaxOutUncorrelated() throws IOException { + setSubsearchMaxOut(1); + JSONObject result = + executeQuery( + String.format( + "source = %s" + + "| where exists [" + + " source = %s | join type=left uid %s" + + " | eval dept = department " + + " | where dept = 'DATA' " + + " ]" + + "| sort - salary" + + "| fields id, name, salary", + TEST_INDEX_WORKER, TEST_INDEX_WORK_INFORMATION, TEST_INDEX_WORK_INFORMATION)); + verifyNumOfRows(result, 7); + resetSubsearchMaxOut(); + } + + @Test + public void testUncorrelatedSubsearchMaxOutZeroMeansUnlimited() throws IOException { + setSubsearchMaxOut(0); + JSONObject result = + executeQuery( + String.format( + "source = %s" + + "| where exists [" + + " source = %s | where name = 'Tom'" + + " ]" + + "| sort - salary" + + "| fields id, name, salary", + TEST_INDEX_WORKER, TEST_INDEX_WORK_INFORMATION)); + verifyNumOfRows(result, 7); + resetSubsearchMaxOut(); + } + + @Test + public void testCorrelatedSubsearchMaxOutZeroMeansUnlimited() throws IOException { + setSubsearchMaxOut(0); + JSONObject result = + executeQuery( + String.format( + "source = %s" + + "| where exists [" + + " source = %s | where id = uid" + + " ]" + + "| sort - salary" + + "| fields id, name, salary", + TEST_INDEX_WORKER, TEST_INDEX_WORK_INFORMATION)); + verifyNumOfRows(result, 5); + result = + executeQuery( + String.format( + "source = %s" + + "| where not exists [" + + " source = %s | where id = uid" + + " ]" + + "| sort - salary" + + "| fields id, name, salary", + TEST_INDEX_WORKER, TEST_INDEX_WORK_INFORMATION)); + verifyNumOfRows(result, 2); + resetSubsearchMaxOut(); + } + + @Test + public void testSubsearchMaxOutNegativeMeansUnlimited() throws IOException { + setSubsearchMaxOut(-1); + JSONObject result = + executeQuery( + String.format( + "source = %s" + + "| where exists [" + + " source = %s | where id = uid" + + " ]" + + "| sort - salary" + + "| fields id, name, salary", + TEST_INDEX_WORKER, TEST_INDEX_WORK_INFORMATION)); + verifyNumOfRows(result, 5); + resetSubsearchMaxOut(); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExplainIT.java index d888833c05d..674a7d96f8d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExplainIT.java @@ -16,7 +16,6 @@ public class CalcitePPLExplainIT extends PPLIntegTestCase { @Override public void init() throws Exception { - GlobalPushdownConfig.enabled = false; super.init(); enableCalcite(); @@ -55,23 +54,6 @@ public void testExplainCommandExtendedWithCodegen() throws IOException { + " org.apache.calcite.DataContext root)")); } - @Test - public void testExplainCommandExtendedWithoutCodegen() throws IOException { - var result = - executeWithReplace("explain extended source=test | where age = 20 | fields name, age"); - if (!isPushdownDisabled()) { - assertFalse( - result.contains( - "public org.apache.calcite.linq4j.Enumerable bind(final" - + " org.apache.calcite.DataContext root)")); - } else { - assertTrue( - result.contains( - "public org.apache.calcite.linq4j.Enumerable bind(final" - + " org.apache.calcite.DataContext root)")); - } - } - @Test public void testExplainCommandCost() throws IOException { var result = executeWithReplace("explain cost source=test | where age = 20 | fields name, age"); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLInSubqueryIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLInSubqueryIT.java index ca97533b4ac..d417421b5a4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLInSubqueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLInSubqueryIT.java @@ -13,6 +13,7 @@ import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRowsInOrder; import static org.opensearch.sql.util.MatcherUtils.verifyErrorMessageContains; +import static org.opensearch.sql.util.MatcherUtils.verifyNumOfRows; import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; @@ -354,4 +355,68 @@ public void testInSubqueryWithTableAlias() throws IOException { verifySchema(result, schema("id", "int"), schema("name", "string"), schema("salary", "int")); verifyDataRowsInOrder(result, rows(1002, "John", 120000), rows(1005, "Jane", 90000)); } + + @Test + public void testInCorrelatedSubquery() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source = %s| where name in [ source = %s | where id = uid and" + + " (like(occupation, '%%ist') or occupation = 'Engineer') | fields name ]|" + + " sort - salary | fields id, name, salary", + TEST_INDEX_WORKER, TEST_INDEX_WORK_INFORMATION)); + verifySchema(result, schema("id", "int"), schema("name", "string"), schema("salary", "int")); + verifyDataRowsInOrder( + result, rows(1002, "John", 120000), rows(1000, "Jake", 100000), rows(1005, "Jane", 90000)); + } + + @Test + public void testSubsearchMaxOut() throws IOException { + setSubsearchMaxOut(1); + JSONObject result = + executeQuery( + String.format( + "source = %s" + + "| where id in [" + + " source = %s | fields uid" + + " ]" + + "| sort - salary" + + "| fields id, name, salary", + TEST_INDEX_WORKER, TEST_INDEX_WORK_INFORMATION)); + verifySchema(result, schema("id", "int"), schema("name", "string"), schema("salary", "int")); + verifyDataRowsInOrder(result, rows(1000, "Jake", 100000)); + resetSubsearchMaxOut(); + } + + @Test + public void testInCorrelatedSubqueryMaxOut() throws IOException { + setSubsearchMaxOut(1); + JSONObject result = + executeQuery( + String.format( + "source = %s| where name in [ source = %s | where id = uid and" + + " (like(occupation, '%%ist') or occupation = 'Engineer') | fields name ]|" + + " sort - salary | fields id, name, salary", + TEST_INDEX_WORKER, TEST_INDEX_WORK_INFORMATION)); + verifyNumOfRows(result, 1); + resetSubsearchMaxOut(); + } + + @Test + public void testSubsearchMaxOutZeroMeansUnlimited() throws IOException { + setSubsearchMaxOut(0); + JSONObject result = + executeQuery( + String.format( + "source = %s" + + "| where id in [" + + " source = %s | fields uid" + + " ]" + + "| sort - salary" + + "| fields id, name, salary", + TEST_INDEX_WORKER, TEST_INDEX_WORK_INFORMATION)); + verifySchema(result, schema("id", "int"), schema("name", "string"), schema("salary", "int")); + verifyNumOfRows(result, 5); + resetSubsearchMaxOut(); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJoinIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJoinIT.java index c9886f687c2..95931157947 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJoinIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJoinIT.java @@ -938,4 +938,22 @@ public void testJoinWithoutFieldListMaxEqualsOne() throws IOException { schema("month", "int")); verifyNumOfRows(actual, 8); } + + @Test + public void testJoinSubsearchMaxOut() throws IOException { + setJoinSubsearchMaxOut(5); + JSONObject actual = + executeQuery( + String.format( + "source=%s | where country = 'Canada' | join type=inner max=0 country %s", + TEST_INDEX_STATE_COUNTRY, TEST_INDEX_OCCUPATION)); + verifyNumOfRows(actual, 10); + resetJoinSubsearchMaxOut(); + actual = + executeQuery( + String.format( + "source=%s | where country = 'Canada' | join type=inner max=0 country %s", + TEST_INDEX_STATE_COUNTRY, TEST_INDEX_OCCUPATION)); + verifyNumOfRows(actual, 15); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java index 0180b70e24a..6cd0674a2dc 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java @@ -53,10 +53,7 @@ public void testRefRenamedField() { String.format( "source = %s | rename age as renamed_age | fields age", TEST_INDEX_STATE_COUNTRY))); - verifyErrorMessageContains( - e, - "field [age] not found; input fields are: [name, country, state, month, year, renamed_age," - + " _id, _index, _score, _maxscore, _sort, _routing]"); + verifyErrorMessageContains(e, "Field [age] not found."); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLScalarSubqueryIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLScalarSubqueryIT.java index 4aac43580f3..133530bbd7b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLScalarSubqueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLScalarSubqueryIT.java @@ -11,6 +11,7 @@ import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifyNumOfRows; import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; @@ -308,4 +309,20 @@ public void testNestedScalarSubqueryWithTableAlias() throws IOException { verifySchema(result, schema("id", "int"), schema("name", "string")); verifyDataRows(result, rows(1000, "Jake")); } + + @Test + public void testSubsearchMaxOutZeroMeansUnlimited() throws IOException { + setSubsearchMaxOut(0); + JSONObject result = + executeQuery( + String.format( + "source = %s" + + "| where id = [" + + " source = %s | where id = uid | stats max(uid)" + + " ]" + + "| fields id, name", + TEST_INDEX_WORKER, TEST_INDEX_WORK_INFORMATION)); + verifyNumOfRows(result, 5); + resetSubsearchMaxOut(); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLStringBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLStringBuiltinFunctionIT.java index 33befc23a50..3e0b6cb07aa 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLStringBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLStringBuiltinFunctionIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.calcite.remote; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY_WITH_NULL; import static org.opensearch.sql.util.MatcherUtils.*; @@ -24,6 +25,7 @@ public void init() throws Exception { loadIndex(Index.STATE_COUNTRY); loadIndex(Index.STATE_COUNTRY_WITH_NULL); + loadIndex(Index.ACCOUNT); } @Test @@ -300,6 +302,77 @@ public void testReplace() throws IOException { verifyDataRows(actual, rows("Jane", 20, "heLLo")); } + @Test + public void testReplaceWithRegexPattern() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | where account_number = 1 | eval street_only = replace(address," + + " '\\\\d+ ', '') | fields address, street_only", + TEST_INDEX_ACCOUNT)); + + verifySchema(actual, schema("address", "string"), schema("street_only", "string")); + + verifyDataRows(actual, rows("880 Holmes Lane", "Holmes Lane")); + } + + @Test + public void testReplaceWithCaptureGroups() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | where account_number = 1 | eval swapped = replace(firstname," + + " '^(.)(.)', '\\\\2\\\\1') | fields firstname, swapped", + TEST_INDEX_ACCOUNT)); + + verifySchema(actual, schema("firstname", "string"), schema("swapped", "string")); + + verifyDataRows(actual, rows("Amber", "mAber")); + } + + @Test + public void testReplaceWithEmailDomainReplacement() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | where account_number = 1 | eval new_email =" + + " replace(email, '([^@]+)@(.+)', '\\\\1@newdomain.com') | fields email," + + " new_email", + TEST_INDEX_ACCOUNT)); + + verifySchema(actual, schema("email", "string"), schema("new_email", "string")); + + verifyDataRows(actual, rows("amberduke@pyrami.com", "amberduke@newdomain.com")); + } + + @Test + public void testReplaceWithCharacterClasses() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | where account_number = 1 | eval masked = replace(address, '[a-zA-Z]'," + + " 'X') | fields address, masked", + TEST_INDEX_ACCOUNT)); + + verifySchema(actual, schema("address", "string"), schema("masked", "string")); + + verifyDataRows(actual, rows("880 Holmes Lane", "880 XXXXXX XXXX")); + } + + @Test + public void testReplaceWithAnchors() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | where account_number = 1 | eval street_name = replace(address," + + " '^\\\\d+\\\\s+', '') | fields address, street_name", + TEST_INDEX_ACCOUNT)); + + verifySchema(actual, schema("address", "string"), schema("street_name", "string")); + + verifyDataRows(actual, rows("880 Holmes Lane", "Holmes Lane")); + } + @Test public void testLeft() throws IOException { JSONObject actual = @@ -326,6 +399,51 @@ public void testStrCmp() throws IOException { verifyDataRows(actual, rows("Jane", 20)); } + @Test + public void testReplaceWithInvalidRegexPattern() { + // Test invalid regex pattern - unclosed character class + Throwable e1 = + assertThrowsWithReplace( + Exception.class, + () -> + executeQuery( + String.format( + "source=%s | eval result = replace(firstname, '[unclosed', 'X') | fields" + + " firstname, result", + TEST_INDEX_ACCOUNT))); + verifyErrorMessageContains(e1, "Invalid regex pattern"); + verifyErrorMessageContains(e1, "Unclosed character class"); + verifyErrorMessageContains(e1, "400 Bad Request"); + + // Test invalid regex pattern - unclosed group + Throwable e2 = + assertThrowsWithReplace( + Exception.class, + () -> + executeQuery( + String.format( + "source=%s | eval result = replace(firstname, '(invalid', 'X') | fields" + + " firstname, result", + TEST_INDEX_ACCOUNT))); + verifyErrorMessageContains(e2, "Invalid regex pattern"); + verifyErrorMessageContains(e2, "Unclosed group"); + verifyErrorMessageContains(e2, "400 Bad Request"); + + // Test invalid regex pattern - dangling metacharacter + Throwable e3 = + assertThrowsWithReplace( + Exception.class, + () -> + executeQuery( + String.format( + "source=%s | eval result = replace(firstname, '?invalid', 'X') | fields" + + " firstname, result", + TEST_INDEX_ACCOUNT))); + verifyErrorMessageContains(e3, "Invalid regex pattern"); + verifyErrorMessageContains(e3, "Dangling meta character"); + verifyErrorMessageContains(e3, "400 Bad Request"); + } + private void prepareTrim() throws IOException { Request request1 = new Request("PUT", "/opensearch-sql_test_index_state_country/_doc/5?refresh=true"); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteParseCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteParseCommandIT.java index 6414a854401..e25470a6e53 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteParseCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteParseCommandIT.java @@ -5,6 +5,10 @@ package org.opensearch.sql.calcite.remote; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; + +import java.io.IOException; +import org.junit.Test; import org.opensearch.sql.ppl.ParseCommandIT; public class CalciteParseCommandIT extends ParseCommandIT { @@ -13,4 +17,60 @@ public void init() throws Exception { super.init(); enableCalcite(); } + + @Test + public void testParseErrorInvalidGroupNameUnderscore() throws IOException { + try { + executeQuery( + String.format( + "source=%s | parse email '.+@(?.+)' | fields email", TEST_INDEX_BANK)); + fail("Should have thrown an exception for underscore in named capture group"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("Invalid capture group name 'host_name'")); + assertTrue( + e.getMessage().contains("must start with a letter and contain only letters and digits")); + } + } + + @Test + public void testParseErrorInvalidGroupNameHyphen() throws IOException { + try { + executeQuery( + String.format( + "source=%s | parse email '.+@(?.+)' | fields email", TEST_INDEX_BANK)); + fail("Should have thrown an exception for hyphen in named capture group"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("Invalid capture group name 'host-name'")); + assertTrue( + e.getMessage().contains("must start with a letter and contain only letters and digits")); + } + } + + @Test + public void testParseErrorInvalidGroupNameStartingWithDigit() throws IOException { + try { + executeQuery( + String.format( + "source=%s | parse email '.+@(?<1host>.+)' | fields email", TEST_INDEX_BANK)); + fail("Should have thrown an exception for group name starting with digit"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("Invalid capture group name '1host'")); + assertTrue( + e.getMessage().contains("must start with a letter and contain only letters and digits")); + } + } + + @Test + public void testParseErrorInvalidGroupNameSpecialCharacter() throws IOException { + try { + executeQuery( + String.format( + "source=%s | parse email '.+@(?.+)' | fields email", TEST_INDEX_BANK)); + fail("Should have thrown an exception for special character in named capture group"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("Invalid capture group name 'host@name'")); + assertTrue( + e.getMessage().contains("must start with a letter and contain only letters and digits")); + } + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteQueryAnalysisIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteQueryAnalysisIT.java index 2d1fb2e5aa7..d6b7e7802a7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteQueryAnalysisIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteQueryAnalysisIT.java @@ -6,9 +6,8 @@ package org.opensearch.sql.calcite.remote; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static org.opensearch.sql.util.MatcherUtils.verifyErrorMessageContains; -import java.io.IOException; -import org.opensearch.client.ResponseException; import org.opensearch.sql.ppl.QueryAnalysisIT; public class CalciteQueryAnalysisIT extends QueryAnalysisIT { @@ -20,16 +19,11 @@ public void init() throws Exception { @Override public void nonexistentFieldShouldFailSemanticCheck() { - String query = String.format("search source=%s | fields name", TEST_INDEX_ACCOUNT); - try { - executeQuery(query); - fail("Expected to throw Exception, but none was thrown for query: " + query); - } catch (ResponseException e) { - String errorMsg = e.getMessage(); - assertTrue(errorMsg.contains("IllegalArgumentException")); - assertTrue(errorMsg.contains("field [name] not found")); - } catch (IOException e) { - throw new IllegalStateException("Unexpected exception raised for query: " + query); - } + Throwable e = + assertThrowsWithReplace( + IllegalArgumentException.class, + () -> + executeQuery(String.format("search source=%s | fields name", TEST_INDEX_ACCOUNT))); + verifyErrorMessageContains(e, "Field [name] not found."); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteRareCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteRareCommandIT.java index eaf59e09a73..9689b7385bb 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteRareCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteRareCommandIT.java @@ -5,6 +5,15 @@ package org.opensearch.sql.calcite.remote; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyNumOfRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchemaInOrder; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.ppl.RareCommandIT; public class CalciteRareCommandIT extends RareCommandIT { @@ -12,5 +21,42 @@ public class CalciteRareCommandIT extends RareCommandIT { public void init() throws Exception { super.init(); enableCalcite(); + loadIndex(Index.BANK_WITH_NULL_VALUES); + } + + @Test + public void testRareCommandUseNull() throws IOException { + JSONObject result = + executeQuery(String.format("source=%s | rare age", TEST_INDEX_BANK_WITH_NULL_VALUES)); + verifySchemaInOrder(result, schema("age", "int"), schema("count", "bigint")); + verifyNumOfRows(result, 6); + } + + @Test + public void testRareCommandUseNullFalse() throws IOException { + JSONObject result = + executeQuery( + String.format("source=%s | rare usenull=false age", TEST_INDEX_BANK_WITH_NULL_VALUES)); + verifySchemaInOrder(result, schema("age", "int"), schema("count", "bigint")); + verifyNumOfRows(result, 5); + } + + @Test + public void testRareCommandLegacyFalse() throws IOException { + withSettings( + Settings.Key.PPL_SYNTAX_LEGACY_PREFERRED, + "false", + () -> { + JSONObject result; + try { + result = + executeQuery( + String.format("source=%s | rare age", TEST_INDEX_BANK_WITH_NULL_VALUES)); + } catch (IOException e) { + throw new RuntimeException(e); + } + verifySchemaInOrder(result, schema("age", "int"), schema("count", "bigint")); + verifyNumOfRows(result, 5); + }); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteReplaceCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteReplaceCommandIT.java new file mode 100644 index 00000000000..44cc4a3aaf0 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteReplaceCommandIT.java @@ -0,0 +1,405 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.legacy.TestsConstants.*; +import static org.opensearch.sql.util.MatcherUtils.*; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +public class CalciteReplaceCommandIT extends PPLIntegTestCase { + + public void init() throws Exception { + super.init(); + enableCalcite(); + disallowCalciteFallback(); + loadIndex(Index.STATE_COUNTRY); + } + + @Test + public void testReplaceWithFields() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source = %s | replace 'USA' WITH 'United States' IN country | fields name, age," + + " country", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema( + result, schema("name", "string"), schema("age", "int"), schema("country", "string")); + + verifyDataRows( + result, + rows("Jake", 70, "United States"), + rows("Hello", 30, "United States"), + rows("John", 25, "Canada"), + rows("Jane", 20, "Canada")); + } + + @Test + public void testMultipleReplace() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source = %s | replace 'USA' WITH 'United States' IN country | replace 'Jane' WITH" + + " 'Joseph' IN name", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema( + result, + schema("name", "string"), + schema("country", "string"), + schema("state", "string"), + schema("month", "int"), + schema("year", "int"), + schema("age", "int")); + + verifyDataRows( + result, + rows("Jake", "United States", "California", 4, 2023, 70), + rows("Hello", "United States", "New York", 4, 2023, 30), + rows("John", "Canada", "Ontario", 4, 2023, 25), + rows("Joseph", "Canada", "Quebec", 4, 2023, 20)); + } + + @Test + public void testReplaceWithSort() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source = %s | replace 'US' WITH 'United States' IN country | sort country", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema( + result, + schema("name", "string"), + schema("age", "int"), + schema("state", "string"), + schema("country", "string"), + schema("year", "int"), + schema("month", "int")); + } + + @Test + public void testReplaceWithWhereClause() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source = %s | where country = 'US' | replace 'US' WITH 'United States' IN country", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema( + result, + schema("name", "string"), + schema("age", "int"), + schema("state", "string"), + schema("country", "string"), + schema("year", "int"), + schema("month", "int")); + } + + @Test + public void testEmptyStringReplacement() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source = %s | replace 'USA' WITH '' IN country", TEST_INDEX_STATE_COUNTRY)); + + verifySchema( + result, + schema("name", "string"), + schema("country", "string"), + schema("state", "string"), + schema("month", "int"), + schema("year", "int"), + schema("age", "int")); + + verifyDataRows( + result, + rows("Jake", "", "California", 4, 2023, 70), + rows("Hello", "", "New York", 4, 2023, 30), + rows("John", "Canada", "Ontario", 4, 2023, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20)); + } + + @Test + public void testMultipleFieldsInClause() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source = %s | replace 'USA' WITH 'United States' IN country,state", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema( + result, + schema("name", "string"), + schema("country", "string"), + schema("state", "string"), + schema("month", "int"), + schema("year", "int"), + schema("age", "int")); + + verifyDataRows( + result, + rows("Jake", "United States", "California", 4, 2023, 70), + rows("Hello", "United States", "New York", 4, 2023, 30), + rows("John", "Canada", "Ontario", 4, 2023, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20)); + } + + @Test + public void testReplaceNonExistentField() { + Throwable e = + assertThrowsWithReplace( + IllegalArgumentException.class, + () -> + executeQuery( + String.format( + "source = %s | replace 'USA' WITH 'United States' IN non_existent_field", + TEST_INDEX_STATE_COUNTRY))); + verifyErrorMessageContains( + e, + "field [non_existent_field] not found; input fields are: [name, country, state, month," + + " year, age, _id, _index, _score, _maxscore, _sort, _routing]"); + } + + @Test + public void testReplaceAfterFieldRemoved() { + Throwable e = + assertThrowsWithReplace( + IllegalArgumentException.class, + () -> + executeQuery( + String.format( + "source = %s | fields name, age | replace 'USA' WITH 'United States' IN" + + " country", + TEST_INDEX_STATE_COUNTRY))); + verifyErrorMessageContains(e, "field [country] not found; input fields are: [name, age]"); + } + + @Test + public void testMissingInClause() { + Throwable e = + assertThrowsWithReplace( + SyntaxCheckException.class, + () -> + executeQuery( + String.format( + "source = %s | replace 'USA' WITH 'United States'", + TEST_INDEX_STATE_COUNTRY))); + + verifyErrorMessageContains(e, "[] is not a valid term at this part of the query"); + verifyErrorMessageContains(e, "Expecting tokens: 'IN'"); + } + + @Test + public void testDuplicateFieldsInReplace() { + Throwable e = + assertThrowsWithReplace( + IllegalArgumentException.class, + () -> + executeQuery( + String.format( + "source = %s | replace 'USA' WITH 'United States' IN country, state," + + " country", + TEST_INDEX_STATE_COUNTRY))); + verifyErrorMessageContains(e, "Duplicate fields [country] in Replace command"); + } + + @Test + public void testNonStringLiteralPattern() { + Throwable e = + assertThrowsWithReplace( + SyntaxCheckException.class, + () -> + executeQuery( + String.format( + "source = %s | replace 23 WITH 'test' IN field1", + TEST_INDEX_STATE_COUNTRY))); + verifyErrorMessageContains(e, "is not a valid term at this part of the query"); + verifyErrorMessageContains(e, "Expecting tokens: DQUOTA_STRING, SQUOTA_STRING"); + } + + @Test + public void testNonStringLiteralReplacement() { + Throwable e = + assertThrowsWithReplace( + SyntaxCheckException.class, + () -> + executeQuery( + String.format( + "source = %s | replace 'test' WITH 45 IN field1", + TEST_INDEX_STATE_COUNTRY))); + verifyErrorMessageContains(e, "is not a valid term at this part of the query"); + verifyErrorMessageContains(e, "Expecting tokens: DQUOTA_STRING, SQUOTA_STRING"); + } + + @Test + public void testMultiplePairsInSingleCommand() throws IOException { + // Test replacing multiple patterns in a single command + JSONObject result = + executeQuery( + String.format( + "source = %s | replace 'USA' WITH 'United States', 'Canada' WITH 'CA' IN country", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema( + result, + schema("name", "string"), + schema("country", "string"), + schema("state", "string"), + schema("month", "int"), + schema("year", "int"), + schema("age", "int")); + + verifyDataRows( + result, + rows("Jake", "United States", "California", 4, 2023, 70), + rows("Hello", "United States", "New York", 4, 2023, 30), + rows("John", "CA", "Ontario", 4, 2023, 25), + rows("Jane", "CA", "Quebec", 4, 2023, 20)); + } + + @Test + public void testMultiplePairsSequentialApplication() throws IOException { + // Test that replacements are applied sequentially (order matters) + // If we have "Ontario" WITH "ON", "ON" WITH "Ontario Province" + // then "Ontario" becomes "ON" first, then that "ON" becomes "Ontario Province" + JSONObject result = + executeQuery( + String.format( + "source = %s | replace 'Ontario' WITH 'ON', 'ON' WITH 'Ontario Province' IN state" + + " | fields name, state", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema(result, schema("name", "string"), schema("state", "string")); + + verifyDataRows( + result, + rows("Jake", "California"), + rows("Hello", "New York"), + rows("John", "Ontario Province"), + rows("Jane", "Quebec")); + } + + @Test + public void testWildcardReplace_suffixMatch() throws IOException { + // Pattern "*ada" should match "Canada" and replace with "CA" + JSONObject result = + executeQuery( + String.format( + "source = %s | replace '*ada' WITH 'CA' IN country | fields name, country", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema(result, schema("name", "string"), schema("country", "string")); + + verifyDataRows( + result, rows("Jake", "USA"), rows("Hello", "USA"), rows("John", "CA"), rows("Jane", "CA")); + } + + @Test + public void testWildcardReplace_prefixMatch() throws IOException { + // Pattern "US*" should match "USA" and replace with "United States" + JSONObject result = + executeQuery( + String.format( + "source = %s | replace 'US*' WITH 'United States' IN country | fields name," + + " country", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema(result, schema("name", "string"), schema("country", "string")); + + verifyDataRows( + result, + rows("Jake", "United States"), + rows("Hello", "United States"), + rows("John", "Canada"), + rows("Jane", "Canada")); + } + + @Test + public void testWildcardReplace_multipleWildcards() throws IOException { + // Pattern "* *" with replacement "*_*" should replace spaces with underscores + JSONObject result = + executeQuery( + String.format( + "source = %s | replace '* *' WITH '*_*' IN state | fields name, state", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema(result, schema("name", "string"), schema("state", "string")); + + verifyDataRows( + result, + rows("Jake", "California"), + rows("Hello", "New_York"), + rows("John", "Ontario"), + rows("Jane", "Quebec")); + } + + @Test + public void testWildcardReplace_symmetryMismatch_shouldFail() { + // Pattern has 2 wildcards, replacement has 1 - should fail + Throwable e = + assertThrowsWithReplace( + IllegalArgumentException.class, + () -> + executeQuery( + String.format( + "source = %s | replace '* *' WITH '*' IN state", + TEST_INDEX_STATE_COUNTRY))); + verifyErrorMessageContains(e, "Wildcard count mismatch"); + } + + @Test + public void testEscapeSequence_literalAsterisk() throws IOException { + // Test matching literal asterisks in data using \* escape sequence + JSONObject result = + executeQuery( + String.format( + "source = %s | eval note = 'price: *sale*' | replace 'price: \\\\*sale\\\\*' WITH" + + " 'DISCOUNTED' IN note | fields note | head 1", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema(result, schema("note", "string")); + // Pattern "price: \*sale\*" matches literal asterisks, result should be "DISCOUNTED" + verifyDataRows(result, rows("DISCOUNTED")); + } + + @Test + public void testEscapeSequence_mixedEscapeAndWildcard() throws IOException { + // Test combining escaped asterisks (literal) with wildcards (pattern matching) + JSONObject result = + executeQuery( + String.format( + "source = %s | eval label = 'file123.txt' | replace 'file*.*' WITH" + + " '\\\\**.*' IN label | fields label | head 1", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema(result, schema("label", "string")); + // Pattern "file*.*" captures "123" and "txt" + // Replacement "\**.*" has escaped * (literal), then 2 wildcards, producing "*123.txt" + verifyDataRows(result, rows("*123.txt")); + } + + @Test + public void testEscapeSequence_noMatchLiteral() throws IOException { + // Test that escaped asterisk doesn't match as wildcard + JSONObject result = + executeQuery( + String.format( + "source = %s | eval test = 'fooXbar' | replace 'foo\\\\*bar' WITH 'matched' IN test" + + " | fields test | head 1", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema(result, schema("test", "string")); + // Pattern "foo\*bar" matches literal "foo*bar", not "fooXbar", so original value returned + verifyDataRows(result, rows("fooXbar")); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteRexCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteRexCommandIT.java index c8facd98cc6..f7a50ee0676 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteRexCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteRexCommandIT.java @@ -50,6 +50,70 @@ public void testRexErrorNoNamedGroups() throws IOException { } } + @Test + public void testRexErrorInvalidGroupNameUnderscore() throws IOException { + try { + executeQuery( + String.format( + "source=%s | rex field=email \\\"(?[^@]+)@(?.+)\\\" | fields" + + " email", + TEST_INDEX_ACCOUNT)); + fail("Should have thrown an exception for underscore in named capture group"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("Invalid capture group name 'user_name'")); + assertTrue( + e.getMessage().contains("must start with a letter and contain only letters and digits")); + } + } + + @Test + public void testRexErrorInvalidGroupNameHyphen() throws IOException { + try { + executeQuery( + String.format( + "source=%s | rex field=email \\\"(?[^@]+)@(?.+)\\\" | fields" + + " email", + TEST_INDEX_ACCOUNT)); + fail("Should have thrown an exception for hyphen in named capture group"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("Invalid capture group name 'user-name'")); + assertTrue( + e.getMessage().contains("must start with a letter and contain only letters and digits")); + } + } + + @Test + public void testRexErrorInvalidGroupNameStartingWithDigit() throws IOException { + try { + executeQuery( + String.format( + "source=%s | rex field=email \\\"(?<1user>[^@]+)@(?.+)\\\" | fields" + + " email", + TEST_INDEX_ACCOUNT)); + fail("Should have thrown an exception for group name starting with digit"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("Invalid capture group name '1user'")); + assertTrue( + e.getMessage().contains("must start with a letter and contain only letters and digits")); + } + } + + @Test + public void testRexErrorInvalidGroupNameSpecialCharacter() throws IOException { + try { + executeQuery( + String.format( + "source=%s | rex field=email \\\"(?[^@]+)@(?.+)\\\" | fields" + + " email", + TEST_INDEX_ACCOUNT)); + fail("Should have thrown an exception for special character in named capture group"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("Invalid capture group name 'user@name'")); + assertTrue( + e.getMessage().contains("must start with a letter and contain only letters and digits")); + } + } + @Test public void testRexWithFiltering() throws IOException { JSONObject result = @@ -242,4 +306,53 @@ public void testRexMaxMatchConfigurableLimit() throws IOException { new ClusterSetting(PERSISTENT, Settings.Key.PPL_REX_MAX_MATCH_LIMIT.getKeyValue(), null)); } } + + @Test + public void testRexNestedCaptureGroupsBugFix() throws IOException { + JSONObject resultWithNested = + executeQuery( + String.format( + "source=%s | rex field=email" + + " \\\"(?[^@]+)@(?(pyrami|gmail|yahoo))\\\\\\\\.(?(com|org|net))\\\"" + + " | fields user, domain, tld | head 1", + TEST_INDEX_ACCOUNT)); + + assertEquals(1, resultWithNested.getJSONArray("datarows").length()); + assertEquals( + "amberduke", + resultWithNested + .getJSONArray("datarows") + .getJSONArray(0) + .get(0)); // user should be "amberduke" + assertEquals( + "pyrami", + resultWithNested + .getJSONArray("datarows") + .getJSONArray(0) + .get(1)); // domain should be "pyrami", NOT "amberduke" + assertEquals( + "com", + resultWithNested + .getJSONArray("datarows") + .getJSONArray(0) + .get(2)); // tld should be "com", NOT "pyrami" + + // More complex nested alternation + JSONObject complexNested = + executeQuery( + String.format( + "source=%s | rex field=firstname" + + " \\\"(?(A|B|C|D|E))[a-z]*(?(ley|nne|ber|ton|son))\\\" |" + + " fields initial, suffix | head 1", + TEST_INDEX_ACCOUNT)); + + if (!complexNested.getJSONArray("datarows").isEmpty()) { + String initial = complexNested.getJSONArray("datarows").getJSONArray(0).getString(0); + String suffix = complexNested.getJSONArray("datarows").getJSONArray(0).getString(1); + + assertTrue("Initial should be a single letter A-E", initial.matches("[A-E]")); + assertTrue( + "Suffix should match alternation pattern", suffix.matches("(ley|nne|ber|ton|son)")); + } + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSortCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSortCommandIT.java index 981b418140e..3e01ba9f9d7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSortCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSortCommandIT.java @@ -24,14 +24,6 @@ public void init() throws Exception { enableCalcite(); } - // TODO: Move this test to SortCommandIT once head-then-sort is fixed in v2. - @Test - public void testHeadThenSort() throws IOException { - JSONObject result = - executeQuery(String.format("source=%s | head 2 | sort age | fields age", TEST_INDEX_BANK)); - verifyOrder(result, rows(32), rows(36)); - } - @Test public void testPushdownSortPlusExpression() throws IOException { String ppl = diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStreamstatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStreamstatsCommandIT.java new file mode 100644 index 00000000000..ee94c218dbb --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStreamstatsCommandIT.java @@ -0,0 +1,1095 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.legacy.TestsConstants.*; +import static org.opensearch.sql.util.MatcherUtils.*; + +import java.io.IOException; +import java.util.List; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +public class CalciteStreamstatsCommandIT extends PPLIntegTestCase { + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + loadIndex(Index.STATE_COUNTRY); + loadIndex(Index.STATE_COUNTRY_WITH_NULL); + loadIndex(Index.BANK_TWO); + loadIndex(Index.LOGS); + } + + @Test + public void testStreamstats() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats count() as cnt, avg(age) as avg, min(age) as min, max(age)" + + " as max", + TEST_INDEX_STATE_COUNTRY)); + + verifySchemaInOrder( + actual, + schema("name", "string"), + schema("country", "string"), + schema("state", "string"), + schema("month", "int"), + schema("year", "int"), + schema("age", "int"), + schema("cnt", "bigint"), + schema("avg", "double"), + schema("min", "int"), + schema("max", "int")); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 1, 70, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 2, 50, 30, 70), + rows("John", "Canada", "Ontario", 4, 2023, 25, 3, 41.666666666666664, 25, 70), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 4, 36.25, 20, 70)); + } + + @Test + public void testStreamstatsWithNull() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats count() as cnt, avg(age) as avg, min(age) as min, max(age)" + + " as max", + TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + + verifySchemaInOrder( + actual, + schema("name", "string"), + schema("country", "string"), + schema("state", "string"), + schema("month", "int"), + schema("year", "int"), + schema("age", "int"), + schema("cnt", "bigint"), + schema("avg", "double"), + schema("min", "int"), + schema("max", "int")); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 1, 70, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 2, 50, 30, 70), + rows("John", "Canada", "Ontario", 4, 2023, 25, 3, 41.666666666666664, 25, 70), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 4, 36.25, 20, 70), + rows(null, "Canada", null, 4, 2023, 10, 5, 31, 10, 70), + rows("Kevin", null, null, 4, 2023, null, 6, 31, 10, 70)); + } + + @Test + public void testStreamstatsBy() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats count() as cnt, avg(age) as avg, min(age) as min, max(age)" + + " as max by country", + TEST_INDEX_STATE_COUNTRY)); + + verifySchemaInOrder( + actual, + schema("name", "string"), + schema("country", "string"), + schema("state", "string"), + schema("month", "int"), + schema("year", "int"), + schema("age", "int"), + schema("cnt", "bigint"), + schema("avg", "double"), + schema("min", "int"), + schema("max", "int")); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 1, 70, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 2, 50, 30, 70), + rows("John", "Canada", "Ontario", 4, 2023, 25, 1, 25, 25, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 2, 22.5, 20, 25)); + } + + @Test + public void testStreamstatsByWithNull() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats count() as cnt, avg(age) as avg, min(age) as min, max(age)" + + " as max by country", + TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + + verifySchemaInOrder( + actual, + schema("name", "string"), + schema("country", "string"), + schema("state", "string"), + schema("month", "int"), + schema("year", "int"), + schema("age", "int"), + schema("cnt", "bigint"), + schema("avg", "double"), + schema("min", "int"), + schema("max", "int")); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 1, 70, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 2, 50, 30, 70), + rows("John", "Canada", "Ontario", 4, 2023, 25, 1, 25, 25, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 2, 22.5, 20, 25), + rows(null, "Canada", null, 4, 2023, 10, 3, 18.333333333333332, 10, 25), + rows("Kevin", null, null, 4, 2023, null, 1, null, null, null)); + + actual = + executeQuery( + String.format( + "source=%s | streamstats count() as cnt, avg(age) as avg, min(age) as min, max(age)" + + " as max by state", + TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 1, 70, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 1, 30, 30, 30), + rows("John", "Canada", "Ontario", 4, 2023, 25, 1, 25, 25, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 1, 20, 20, 20), + rows(null, "Canada", null, 4, 2023, 10, 1, 10, 10, 10), + rows("Kevin", null, null, 4, 2023, null, 2, 10, 10, 10)); + } + + @Test + public void testStreamstatsBySpan() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats count() as cnt, avg(age) as avg, min(age) as min, max(age)" + + " as max by span(age, 10) as age_span", + TEST_INDEX_STATE_COUNTRY)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 1, 70, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 1, 30, 30, 30), + rows("John", "Canada", "Ontario", 4, 2023, 25, 1, 25, 25, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 2, 22.5, 20, 25)); + } + + @Test + public void testStreamstatsBySpanWithNull() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats count() as cnt, avg(age) as avg, min(age) as min, max(age)" + + " as max by span(age, 10) as age_span", + TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 1, 70, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 1, 30, 30, 30), + rows("John", "Canada", "Ontario", 4, 2023, 25, 1, 25, 25, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 2, 22.5, 20, 25), + rows(null, "Canada", null, 4, 2023, 10, 1, 10, 10, 10), + rows("Kevin", null, null, 4, 2023, null, 1, null, null, null)); + } + + @Test + public void testStreamstatsByMultiplePartitions1() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats count() as cnt, avg(age) as avg, min(age) as min, max(age)" + + " as max by span(age, 10) as age_span, country", + TEST_INDEX_STATE_COUNTRY)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 1, 70, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 1, 30, 30, 30), + rows("John", "Canada", "Ontario", 4, 2023, 25, 1, 25, 25, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 2, 22.5, 20, 25)); + } + + @Test + public void testStreamstatsByMultiplePartitions2() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats count() as cnt, avg(age) as avg, min(age) as min, max(age)" + + " as max by span(age, 10) as age_span, state", + TEST_INDEX_STATE_COUNTRY)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 1, 70, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 1, 30, 30, 30), + rows("John", "Canada", "Ontario", 4, 2023, 25, 1, 25, 25, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 1, 20, 20, 20)); + } + + @Test + public void testStreamstatsByMultiplePartitionsWithNull1() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats count() as cnt, avg(age) as avg, min(age) as min, max(age)" + + " as max by span(age, 10) as age_span, country", + TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 1, 70, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 1, 30, 30, 30), + rows("John", "Canada", "Ontario", 4, 2023, 25, 1, 25, 25, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 2, 22.5, 20, 25), + rows(null, "Canada", null, 4, 2023, 10, 1, 10, 10, 10), + rows("Kevin", null, null, 4, 2023, null, 1, null, null, null)); + } + + @Test + public void testStreamstatsByMultiplePartitionsWithNull2() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats count() as cnt, avg(age) as avg, min(age) as min, max(age)" + + " as max by span(age, 10) as age_span, state", + TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 1, 70, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 1, 30, 30, 30), + rows("John", "Canada", "Ontario", 4, 2023, 25, 1, 25, 25, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 1, 20, 20, 20), + rows(null, "Canada", null, 4, 2023, 10, 1, 10, 10, 10), + rows("Kevin", null, null, 4, 2023, null, 1, null, null, null)); + } + + @Test + public void testStreamstatsCurrent() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats current=false avg(age) as prev_avg", + TEST_INDEX_STATE_COUNTRY)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, null), + rows("Hello", "USA", "New York", 4, 2023, 30, 70), + rows("John", "Canada", "Ontario", 4, 2023, 25, 50), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 41.666666666666664)); + } + + @Test + public void testStreamstatsCurrentWithNUll() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats current=false avg(age) as prev_avg", + TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, null), + rows("Hello", "USA", "New York", 4, 2023, 30, 70), + rows("John", "Canada", "Ontario", 4, 2023, 25, 50), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 41.666666666666664), + rows(null, "Canada", null, 4, 2023, 10, 36.25), + rows("Kevin", null, null, 4, 2023, null, 31)); + } + + @Test + public void testStreamstatsWindow() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats window = 3 avg(age) as avg", TEST_INDEX_STATE_COUNTRY)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 50), + rows("John", "Canada", "Ontario", 4, 2023, 25, 41.666666666666664), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 25)); + } + + @Test + public void testStreamstatsWindowWithNull() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats window = 3 avg(age) as avg", + TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 50), + rows("John", "Canada", "Ontario", 4, 2023, 25, 41.666666666666664), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 25), + rows(null, "Canada", null, 4, 2023, 10, 18.333333333333332), + rows("Kevin", null, null, 4, 2023, null, 15)); + } + + public void testStreamstatsBigWindow() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats window = 10 avg(age) as avg", TEST_INDEX_STATE_COUNTRY)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 50), + rows("John", "Canada", "Ontario", 4, 2023, 25, 41.666666666666664), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 36.25)); + } + + @Test + public void testStreamstatsWindowError() { + Throwable e = + assertThrowsWithReplace( + IllegalArgumentException.class, + () -> + executeQuery( + String.format( + "source=%s | streamstats window=-1 avg(age) as avg", + TEST_INDEX_STATE_COUNTRY))); + verifyErrorMessageContains(e, "Window size must be >= 0, but got: -1"); + } + + @Test + public void testStreamstatsCurrentAndWindow() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats current = false window = 2 avg(age) as avg", + TEST_INDEX_STATE_COUNTRY)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, null), + rows("Hello", "USA", "New York", 4, 2023, 30, 70), + rows("John", "Canada", "Ontario", 4, 2023, 25, 50), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 27.5)); + } + + @Test + public void testStreamstatsCurrentAndWindowWithNull() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats current = false window = 2 avg(age) as avg", + TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, null), + rows("Hello", "USA", "New York", 4, 2023, 30, 70), + rows("John", "Canada", "Ontario", 4, 2023, 25, 50), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 27.5), + rows(null, "Canada", null, 4, 2023, 10, 22.5), + rows("Kevin", null, null, 4, 2023, null, 15)); + } + + @Test + public void testStreamstatsGlobal() throws IOException { + final int docId = 5; + Request insertRequest = + new Request( + "PUT", String.format("/%s/_doc/%d?refresh=true", TEST_INDEX_STATE_COUNTRY, docId)); + insertRequest.setJsonEntity( + "{\"name\": \"Jay\",\"age\": 40,\"state\":" + + " \"Quebec\",\"country\": \"USA\",\"year\": 2023,\"month\":" + + " 4}\n"); + client().performRequest(insertRequest); + + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats window=2 global=false avg(age) as avg by country", + TEST_INDEX_STATE_COUNTRY)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 50), + rows("John", "Canada", "Ontario", 4, 2023, 25, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 22.5), + rows("Jay", "USA", "Quebec", 4, 2023, 40, 35)); + + JSONObject actual2 = + executeQuery( + String.format( + "source=%s | streamstats window=2 global=true avg(age) as avg by country", + TEST_INDEX_STATE_COUNTRY)); + + verifyDataRows( + actual2, + rows("Jake", "USA", "California", 4, 2023, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 50), + rows("John", "Canada", "Ontario", 4, 2023, 25, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 22.5), + rows("Jay", "USA", "Quebec", 4, 2023, 40, 40)); + + Request deleteRequest = + new Request( + "DELETE", String.format("/%s/_doc/%d?refresh=true", TEST_INDEX_STATE_COUNTRY, docId)); + client().performRequest(deleteRequest); + } + + @Test + public void testStreamstatsGlobalWithNull() throws IOException { + final int docId = 7; + Request insertRequest = + new Request( + "PUT", + String.format("/%s/_doc/%d?refresh=true", TEST_INDEX_STATE_COUNTRY_WITH_NULL, docId)); + insertRequest.setJsonEntity( + "{\"name\": \"Jay\",\"age\": 40,\"state\":" + + " \"Quebec\",\"country\": \"USA\",\"year\": 2023,\"month\":" + + " 4}\n"); + client().performRequest(insertRequest); + + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats window=2 global=false avg(age) as avg by country", + TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 50), + rows("John", "Canada", "Ontario", 4, 2023, 25, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 22.5), + rows(null, "Canada", null, 4, 2023, 10, 15), + rows("Kevin", null, null, 4, 2023, null, null), + rows("Jay", "USA", "Quebec", 4, 2023, 40, 35)); + + JSONObject actual2 = + executeQuery( + String.format( + "source=%s | streamstats window=2 global=true avg(age) as avg by country", + TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + + verifyDataRows( + actual2, + rows("Jake", "USA", "California", 4, 2023, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 50), + rows("John", "Canada", "Ontario", 4, 2023, 25, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 22.5), + rows(null, "Canada", null, 4, 2023, 10, 15), + rows("Kevin", null, null, 4, 2023, null, null), + rows("Jay", "USA", "Quebec", 4, 2023, 40, 40)); + + Request deleteRequest = + new Request( + "DELETE", + String.format("/%s/_doc/%d?refresh=true", TEST_INDEX_STATE_COUNTRY_WITH_NULL, docId)); + client().performRequest(deleteRequest); + } + + @Test + public void testStreamstatsReset() throws IOException { + final int docId = 5; + Request insertRequest = + new Request( + "PUT", String.format("/%s/_doc/%d?refresh=true", TEST_INDEX_STATE_COUNTRY, docId)); + insertRequest.setJsonEntity( + "{\"name\": \"Jay\",\"age\": 28,\"state\":" + + " \"Quebec\",\"country\": \"USA\",\"year\": 2023,\"month\":" + + " 4}\n"); + client().performRequest(insertRequest); + + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats window=2 reset_before=age>29 avg(age) as avg by country", + TEST_INDEX_STATE_COUNTRY)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 30), + rows("John", "Canada", "Ontario", 4, 2023, 25, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 22.5), + rows("Jay", "USA", "Quebec", 4, 2023, 28, 28)); + + JSONObject actual2 = + executeQuery( + String.format( + "source=%s | streamstats window=2 reset_after=age>22 avg(age) as avg by country", + TEST_INDEX_STATE_COUNTRY)); + + verifyDataRows( + actual2, + rows("Jake", "USA", "California", 4, 2023, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 30), + rows("John", "Canada", "Ontario", 4, 2023, 25, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 20), + rows("Jay", "USA", "Quebec", 4, 2023, 28, 28)); + + Request deleteRequest = + new Request( + "DELETE", String.format("/%s/_doc/%d?refresh=true", TEST_INDEX_STATE_COUNTRY, docId)); + client().performRequest(deleteRequest); + } + + @Test + public void testStreamstatsResetWithNull() throws IOException { + final int docId = 7; + Request insertRequest = + new Request( + "PUT", + String.format("/%s/_doc/%d?refresh=true", TEST_INDEX_STATE_COUNTRY_WITH_NULL, docId)); + insertRequest.setJsonEntity( + "{\"name\": \"Jay\",\"age\": 28,\"state\":" + + " \"Quebec\",\"country\": \"USA\",\"year\": 2023,\"month\":" + + " 4}\n"); + client().performRequest(insertRequest); + + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats window=2 reset_before=age>29 avg(age) as avg by country", + TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 30), + rows("John", "Canada", "Ontario", 4, 2023, 25, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 22.5), + rows(null, "Canada", null, 4, 2023, 10, 15), + rows("Kevin", null, null, 4, 2023, null, null), + rows("Jay", "USA", "Quebec", 4, 2023, 28, 28)); + + JSONObject actual2 = + executeQuery( + String.format( + "source=%s | streamstats window=2 reset_after=age>22 avg(age) as avg by country", + TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + + verifyDataRows( + actual2, + rows("Jake", "USA", "California", 4, 2023, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 30), + rows("John", "Canada", "Ontario", 4, 2023, 25, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 20), + rows(null, "Canada", null, 4, 2023, 10, 15), + rows("Kevin", null, null, 4, 2023, null, null), + rows("Jay", "USA", "Quebec", 4, 2023, 28, 28)); + + Request deleteRequest = + new Request( + "DELETE", + String.format("/%s/_doc/%d?refresh=true", TEST_INDEX_STATE_COUNTRY_WITH_NULL, docId)); + client().performRequest(deleteRequest); + } + + @Test + public void testUnsupportedWindowFunctions() { + List unsupported = List.of("PERCENTILE_APPROX", "PERCENTILE"); + for (String u : unsupported) { + Throwable e = + assertThrowsWithReplace( + UnsupportedOperationException.class, + () -> + executeQuery( + String.format( + "source=%s | streamstats %s(age)", TEST_INDEX_STATE_COUNTRY, u))); + verifyErrorMessageContains(e, "Unexpected window function: " + u); + } + } + + @Test + public void testMultipleStreamstats() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats avg(age) as avg_age by state, country | streamstats" + + " avg(avg_age) as avg_state_age by country", + TEST_INDEX_STATE_COUNTRY)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 30, 50), + rows("John", "Canada", "Ontario", 4, 2023, 25, 25, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 20, 22.5)); + } + + @Test + public void testMultipleStreamstatsWithNull() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats avg(age) as avg_age by state, country | streamstats" + + " avg(avg_age) as avg_state_age by country", + TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 30, 50), + rows("John", "Canada", "Ontario", 4, 2023, 25, 25, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 20, 22.5), + rows(null, "Canada", null, 4, 2023, 10, 10, 18.333333333333332), + rows("Kevin", null, null, 4, 2023, null, null, null)); + } + + @Test + public void testStreamstatsAndEventstats() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eventstats avg(age) as avg_age| streamstats" + + " avg(age) as avg_age_stream", + TEST_INDEX_STATE_COUNTRY)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 36.25, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 36.25, 50), + rows("John", "Canada", "Ontario", 4, 2023, 25, 36.25, 41.666666666666664), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 36.25, 36.25)); + } + + @Test + public void testStreamstatsAndSort() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | sort age | streamstats window = 2 avg(age) as avg_age ", + TEST_INDEX_STATE_COUNTRY)); + + verifyDataRows( + actual, + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 20), + rows("John", "Canada", "Ontario", 4, 2023, 25, 22.5), + rows("Hello", "USA", "New York", 4, 2023, 30, 27.5), + rows("Jake", "USA", "California", 4, 2023, 70, 50)); + } + + @Test + public void testLeftJoinWithStreamstats() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s as l | left join left=l right=r on l.country = r.country [ source=%s |" + + " streamstats window=2 avg(age) as avg_age]", + TEST_INDEX_STATE_COUNTRY, TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + + verifyDataRows( + actual, + rows( + "John", "Canada", "Ontario", 4, 2023, 25, "John", "Canada", "Ontario", 4, 2023, 25, + 27.5), + rows( + "John", "Canada", "Ontario", 4, 2023, 25, "Jane", "Canada", "Quebec", 4, 2023, 20, + 22.5), + rows("John", "Canada", "Ontario", 4, 2023, 25, null, "Canada", null, 4, 2023, 10, 15), + rows( + "Jane", "Canada", "Quebec", 4, 2023, 20, "John", "Canada", "Ontario", 4, 2023, 25, + 27.5), + rows( + "Jane", "Canada", "Quebec", 4, 2023, 20, "Jane", "Canada", "Quebec", 4, 2023, 20, 22.5), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, null, "Canada", null, 4, 2023, 10, 15), + rows( + "Jake", "USA", "California", 4, 2023, 70, "Jake", "USA", "California", 4, 2023, 70, 70), + rows("Jake", "USA", "California", 4, 2023, 70, "Hello", "USA", "New York", 4, 2023, 30, 50), + rows("Hello", "USA", "New York", 4, 2023, 30, "Jake", "USA", "California", 4, 2023, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, "Hello", "USA", "New York", 4, 2023, 30, 50)); + } + + @Test + public void testWhereInWithStreamstatsSubquery() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | where country in [ source=%s | streamstats window=2 avg(age) as" + + " avg_age | where avg_age > 40 | fields country ]", + TEST_INDEX_STATE_COUNTRY, TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70), + rows("Hello", "USA", "New York", 4, 2023, 30)); + } + + @Test + public void testMultipleStreamstatsWithEval() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats avg(age) as avg_age by country, state, name | eval" + + " avg_age_divide_20 = avg_age - 20 | streamstats avg(avg_age_divide_20) as" + + " avg_state_age by country, state | where avg_state_age > 0 | streamstats" + + " count(avg_state_age) as count_country_age_greater_20 by country", + TEST_INDEX_STATE_COUNTRY)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 70, 50, 50, 1), + rows("Hello", "USA", "New York", 4, 2023, 30, 30, 10, 10, 2), + rows("John", "Canada", "Ontario", 4, 2023, 25, 25, 5, 5, 1)); + } + + @Test + public void testStreamstatsEmptyRows() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | where name = 'non-existed' | streamstats count(), avg(age), min(age)," + + " max(age), stddev_pop(age), stddev_samp(age), var_pop(age), var_samp(age)", + TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + verifyNumOfRows(actual, 0); + + JSONObject actual2 = + executeQuery( + String.format( + "source=%s | where name = 'non-existed' | streamstats count(), avg(age), min(age)," + + " max(age), stddev_pop(age), stddev_samp(age), var_pop(age), var_samp(age) by" + + " country", + TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + verifyNumOfRows(actual2, 0); + } + + @Test + public void testStreamstatsVariance() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats stddev_pop(age), stddev_samp(age), var_pop(age)," + + " var_samp(age)", + TEST_INDEX_STATE_COUNTRY)); + + verifySchemaInOrder( + actual, + schema("name", "string"), + schema("country", "string"), + schema("state", "string"), + schema("month", "int"), + schema("year", "int"), + schema("age", "int"), + schema("stddev_pop(age)", "double"), + schema("stddev_samp(age)", "double"), + schema("var_pop(age)", "double"), + schema("var_samp(age)", "double")); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 0, null, 0, null), + rows("Hello", "USA", "New York", 4, 2023, 30, 20, 28.284271247461902, 400, 800), + rows( + "John", + "Canada", + "Ontario", + 4, + 2023, + 25, + 20.138409955990955, + 24.66441431158124, + 405.55555555555566, + 608.3333333333335), + rows( + "Jane", + "Canada", + "Quebec", + 4, + 2023, + 20, + 19.803724397193573, + 22.86737122335374, + 392.1875, + 522.9166666666666)); + } + + @Test + public void testStreamstatsVarianceWithNull() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats stddev_pop(age), stddev_samp(age), var_pop(age)," + + " var_samp(age)", + TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + + verifySchemaInOrder( + actual, + schema("name", "string"), + schema("country", "string"), + schema("state", "string"), + schema("month", "int"), + schema("year", "int"), + schema("age", "int"), + schema("stddev_pop(age)", "double"), + schema("stddev_samp(age)", "double"), + schema("var_pop(age)", "double"), + schema("var_samp(age)", "double")); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 0, null, 0, null), + rows("Hello", "USA", "New York", 4, 2023, 30, 20, 28.284271247461902, 400, 800), + rows( + "John", + "Canada", + "Ontario", + 4, + 2023, + 25, + 20.138409955990955, + 24.66441431158124, + 405.55555555555566, + 608.3333333333335), + rows( + "Jane", + "Canada", + "Quebec", + 4, + 2023, + 20, + 19.803724397193573, + 22.86737122335374, + 392.1875, + 522.9166666666666), + rows(null, "Canada", null, 4, 2023, 10, 20.591260281974, 23.021728866442675, 424, 530), + rows("Kevin", null, null, 4, 2023, null, 20.591260281974, 23.021728866442675, 424, 530)); + } + + @Test + public void testStreamstatsVarianceBy() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats stddev_pop(age), stddev_samp(age), var_pop(age)," + + " var_samp(age) by country", + TEST_INDEX_STATE_COUNTRY)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 0, null, 0, null), + rows("Hello", "USA", "New York", 4, 2023, 30, 20, 28.284271247461902, 400, 800), + rows("John", "Canada", "Ontario", 4, 2023, 25, 0, null, 0, null), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 2.5, 3.5355339059327378, 6.25, 12.5)); + } + + @Test + public void testStreamstatsVarianceBySpan() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | where country != 'USA' | streamstats stddev_samp(age) by span(age," + + " 10)", + TEST_INDEX_STATE_COUNTRY)); + + verifyDataRows( + actual, + rows("John", "Canada", "Ontario", 4, 2023, 25, null), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 3.5355339059327378)); + } + + @Test + public void testStreamstatsVarianceWithNullBy() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats stddev_pop(age), stddev_samp(age), var_pop(age)," + + " var_samp(age) by country", + TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 0, null, 0, null), + rows("Hello", "USA", "New York", 4, 2023, 30, 20, 28.284271247461902, 400, 800), + rows("John", "Canada", "Ontario", 4, 2023, 25, 0, null, 0, null), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 2.5, 3.5355339059327378, 6.25, 12.5), + rows( + null, + "Canada", + null, + 4, + 2023, + 10, + 6.2360956446232345, + 7.6376261582597325, + 38.88888888888888, + 58.333333333333314), + rows("Kevin", null, null, 4, 2023, null, null, null, null, null)); + } + + @Test + public void testStreamstatsDistinctCount() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats dc(state) as dc_state", TEST_INDEX_STATE_COUNTRY)); + + verifySchemaInOrder( + actual, + schema("name", "string"), + schema("country", "string"), + schema("state", "string"), + schema("month", "int"), + schema("year", "int"), + schema("age", "int"), + schema("dc_state", "bigint")); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 1), + rows("Hello", "USA", "New York", 4, 2023, 30, 2), + rows("John", "Canada", "Ontario", 4, 2023, 25, 3), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 4)); + } + + @Test + public void testStreamstatsDistinctCountByCountry() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats dc(state) as dc_state by country", + TEST_INDEX_STATE_COUNTRY)); + + verifySchemaInOrder( + actual, + schema("name", "string"), + schema("country", "string"), + schema("state", "string"), + schema("month", "int"), + schema("year", "int"), + schema("age", "int"), + schema("dc_state", "bigint")); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 1), + rows("Hello", "USA", "New York", 4, 2023, 30, 2), + rows("John", "Canada", "Ontario", 4, 2023, 25, 1), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 2)); + } + + @Test + public void testStreamstatsDistinctCountFunction() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats distinct_count(country) as dc_country", + TEST_INDEX_STATE_COUNTRY)); + + verifySchemaInOrder( + actual, + schema("name", "string"), + schema("country", "string"), + schema("state", "string"), + schema("month", "int"), + schema("year", "int"), + schema("age", "int"), + schema("dc_country", "bigint")); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 1), + rows("Hello", "USA", "New York", 4, 2023, 30, 1), + rows("John", "Canada", "Ontario", 4, 2023, 25, 2), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 2)); + } + + @Test + public void testStreamstatsDistinctCountWithNull() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats dc(state) as dc_state", + TEST_INDEX_STATE_COUNTRY_WITH_NULL)); + + verifySchemaInOrder( + actual, + schema("name", "string"), + schema("country", "string"), + schema("state", "string"), + schema("month", "int"), + schema("year", "int"), + schema("age", "int"), + schema("dc_state", "bigint")); + + verifyDataRows( + actual, + rows("Jake", "USA", "California", 4, 2023, 70, 1), + rows("Hello", "USA", "New York", 4, 2023, 30, 2), + rows("John", "Canada", "Ontario", 4, 2023, 25, 3), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 4), + rows(null, "Canada", null, 4, 2023, 10, 4), + rows("Kevin", null, null, 4, 2023, null, 4)); + } + + @Test + public void testStreamstatsEarliestAndLatest() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | streamstats earliest(message), latest(message) by server", + TEST_INDEX_LOGS)); + verifySchema( + actual, + schema("created_at", "timestamp"), + schema("server", "string"), + schema("@timestamp", "timestamp"), + schema("message", "string"), + schema("level", "string"), + schema("earliest(message)", "string"), + schema("latest(message)", "string")); + verifyDataRows( + actual, + rows( + "2023-01-05 00:00:00", + "server1", + "2023-01-01 00:00:00", + "Database connection failed", + "ERROR", + "Database connection failed", + "Database connection failed"), + rows( + "2023-01-04 00:00:00", + "server2", + "2023-01-02 00:00:00", + "Service started", + "INFO", + "Service started", + "Service started"), + rows( + "2023-01-03 00:00:00", + "server1", + "2023-01-03 00:00:00", + "High memory usage", + "WARN", + "Database connection failed", + "High memory usage"), + rows( + "2023-01-02 00:00:00", + "server3", + "2023-01-04 00:00:00", + "Disk space low", + "ERROR", + "Disk space low", + "Disk space low"), + rows( + "2023-01-01 00:00:00", + "server2", + "2023-01-05 00:00:00", + "Backup completed", + "INFO", + "Service started", + "Backup completed")); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTimechartPerFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTimechartPerFunctionIT.java new file mode 100644 index 00000000000..41751376424 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTimechartPerFunctionIT.java @@ -0,0 +1,211 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +public class CalciteTimechartPerFunctionIT extends PPLIntegTestCase { + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + disallowCalciteFallback(); + + loadIndex(Index.EVENTS_TRAFFIC); + } + + @Test + public void testTimechartPerSecondWithDefaultSpan() throws IOException { + JSONObject result = + executeQuery( + "source=events_traffic | where month(@timestamp) = 9 | timechart per_second(packets)"); + + verifySchema( + result, schema("@timestamp", "timestamp"), schema("per_second(packets)", "double")); + verifyDataRows( + result, + rows("2025-09-08 10:00:00", 1.0), // 60 / 1m + rows("2025-09-08 10:01:00", 2.0), // 120 / 1m + rows("2025-09-08 10:02:00", 4.0)); // (60+180) / 1m + } + + @Test + public void testTimechartPerSecondWithSpecifiedSpan() throws IOException { + JSONObject result = + executeQuery( + "source=events_traffic | where month(@timestamp) = 9 | timechart span=2m" + + " per_second(packets)"); + + verifySchema( + result, schema("@timestamp", "timestamp"), schema("per_second(packets)", "double")); + verifyDataRows( + result, + rows("2025-09-08 10:00:00", 1.5), // (60+120) / 2m + rows("2025-09-08 10:02:00", 2.0)); // (60+180) / 2m + } + + @Test + public void testTimechartPerSecondWithByClause() throws IOException { + JSONObject result = + executeQuery( + "source=events_traffic | where month(@timestamp) = 9 | timechart span=2m" + + " per_second(packets) by host"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("host", "string"), + schema("per_second(packets)", "double")); + verifyDataRows( + result, + rows("2025-09-08 10:00:00", "server1", 1.5), // (60+120) / 2m + rows("2025-09-08 10:02:00", "server1", 0.5), // 60 / 2m + rows("2025-09-08 10:02:00", "server2", 1.5)); // 180 / 2m + } + + @Test + public void testTimechartPerSecondWithLimitAndByClause() throws IOException { + JSONObject result = + executeQuery( + "source=events_traffic | where month(@timestamp) = 9 | timechart span=2m limit=1" + + " per_second(packets) by host"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("host", "string"), + schema("per_second(packets)", "double")); + verifyDataRows( + result, + rows("2025-09-08 10:00:00", "server1", 1.5), + rows("2025-09-08 10:02:00", "server1", 0.5), + rows("2025-09-08 10:02:00", "OTHER", 1.5)); + } + + @Test + public void testTimechartPerSecondWithVariableMonthLengths() throws IOException { + JSONObject result = + executeQuery( + "source=events_traffic | where month(@timestamp) != 9 | timechart span=1M" + + " per_second(packets)"); + + verifySchema( + result, schema("@timestamp", "timestamp"), schema("per_second(packets)", "double")); + verifyDataRows( + result, + rows("2025-02-01 00:00:00", 7.75), // 18748800 / 28 days' seconds + rows("2025-10-01 00:00:00", 7.0)); // 18748800 / 31 days' seconds + } + + @Test + public void testTimechartPerMinuteWithSpecifiedSpan() throws IOException { + JSONObject result = + executeQuery( + "source=events_traffic | where month(@timestamp) = 9 | timechart span=2m" + + " per_minute(packets)"); + + verifySchema( + result, schema("@timestamp", "timestamp"), schema("per_minute(packets)", "double")); + verifyDataRows( + result, + rows("2025-09-08 10:00:00", 90.0), // (60+120) / 2m + rows("2025-09-08 10:02:00", 120.0)); // (60+180) / 2m + } + + @Test + public void testTimechartPerMinuteWithByClause() throws IOException { + JSONObject result = + executeQuery( + "source=events_traffic | where month(@timestamp) = 9 | timechart span=2m" + + " per_minute(packets) by host"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("host", "string"), + schema("per_minute(packets)", "double")); + verifyDataRows( + result, + rows("2025-09-08 10:00:00", "server1", 90.0), // (60+120) / 2m + rows("2025-09-08 10:02:00", "server1", 30.0), // 60 / 2m + rows("2025-09-08 10:02:00", "server2", 90.0)); // 180 / 2m + } + + @Test + public void testTimechartPerHourWithSpecifiedSpan() throws IOException { + JSONObject result = + executeQuery( + "source=events_traffic | where month(@timestamp) = 9 | timechart span=2m" + + " per_hour(packets)"); + + verifySchema(result, schema("@timestamp", "timestamp"), schema("per_hour(packets)", "double")); + verifyDataRows( + result, + rows("2025-09-08 10:00:00", 5400.0), // (60+120) * 30 + rows("2025-09-08 10:02:00", 7200.0)); // (60+180) * 30 + } + + @Test + public void testTimechartPerHourWithByClause() throws IOException { + JSONObject result = + executeQuery( + "source=events_traffic | where month(@timestamp) = 9 | timechart span=2m" + + " per_hour(packets) by host"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("host", "string"), + schema("per_hour(packets)", "double")); + verifyDataRows( + result, + rows("2025-09-08 10:00:00", "server1", 5400.0), // (60+120) * 30 + rows("2025-09-08 10:02:00", "server1", 1800.0), // 60 * 30 + rows("2025-09-08 10:02:00", "server2", 5400.0)); // 180 * 30 + } + + @Test + public void testTimechartPerDayWithSpecifiedSpan() throws IOException { + JSONObject result = + executeQuery( + "source=events_traffic | where month(@timestamp) = 9 | timechart span=2m" + + " per_day(packets)"); + + verifySchema(result, schema("@timestamp", "timestamp"), schema("per_day(packets)", "double")); + verifyDataRows( + result, + rows("2025-09-08 10:00:00", 129600.0), // (60+120) * 720 + rows("2025-09-08 10:02:00", 172800.0)); // (60+180) * 720 + } + + @Test + public void testTimechartPerDayWithByClause() throws IOException { + JSONObject result = + executeQuery( + "source=events_traffic | where month(@timestamp) = 9 | timechart span=2m" + + " per_day(packets) by host"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("host", "string"), + schema("per_day(packets)", "double")); + verifyDataRows( + result, + rows("2025-09-08 10:00:00", "server1", 129600.0), // (60+120) * 720 + rows("2025-09-08 10:02:00", "server1", 43200.0), // 60 * 720 + rows("2025-09-08 10:02:00", "server2", 129600.0)); // 180 * 720 + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTopCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTopCommandIT.java index 76a8b1e49cf..e555576a9cd 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTopCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTopCommandIT.java @@ -5,6 +5,15 @@ package org.opensearch.sql.calcite.remote; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyNumOfRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchemaInOrder; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.ppl.TopCommandIT; public class CalciteTopCommandIT extends TopCommandIT { @@ -12,5 +21,42 @@ public class CalciteTopCommandIT extends TopCommandIT { public void init() throws Exception { super.init(); enableCalcite(); + loadIndex(Index.BANK_WITH_NULL_VALUES); + } + + @Test + public void testTopCommandUseNull() throws IOException { + JSONObject result = + executeQuery(String.format("source=%s | top age", TEST_INDEX_BANK_WITH_NULL_VALUES)); + verifySchemaInOrder(result, schema("age", "int"), schema("count", "bigint")); + verifyNumOfRows(result, 6); + } + + @Test + public void testTopCommandUseNullFalse() throws IOException { + JSONObject result = + executeQuery( + String.format("source=%s | top usenull=false age", TEST_INDEX_BANK_WITH_NULL_VALUES)); + verifySchemaInOrder(result, schema("age", "int"), schema("count", "bigint")); + verifyNumOfRows(result, 5); + } + + @Test + public void testTopCommandLegacyFalse() throws IOException { + withSettings( + Settings.Key.PPL_SYNTAX_LEGACY_PREFERRED, + "false", + () -> { + JSONObject result; + try { + result = + executeQuery( + String.format("source=%s | top age", TEST_INDEX_BANK_WITH_NULL_VALUES)); + } catch (IOException e) { + throw new RuntimeException(e); + } + verifySchemaInOrder(result, schema("age", "int"), schema("count", "bigint")); + verifyNumOfRows(result, 5); + }); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLIntegTestCase.java index 6bc3c07f69a..a607dc39f2b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLIntegTestCase.java @@ -112,6 +112,7 @@ private Settings defaultSettings() { private final Map defaultSettings = new ImmutableMap.Builder() .put(Key.QUERY_SIZE_LIMIT, 200) + .put(Key.QUERY_BUCKET_SIZE, 1000) .put(Key.SQL_CURSOR_KEEP_ALIVE, TimeValue.timeValueMinutes(1)) .put(Key.FIELD_TYPE_TOLERANCE, true) .put(Key.CALCITE_ENGINE_ENABLED, true) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLRelNodeIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLRelNodeIntegTestCase.java new file mode 100644 index 00000000000..9c7d8c90ac3 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLRelNodeIntegTestCase.java @@ -0,0 +1,121 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.standalone; + +import java.io.IOException; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; +import org.apache.calcite.plan.Contexts; +import org.apache.calcite.plan.RelTraitDef; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.Programs; +import org.apache.calcite.tools.RelBuilder; +import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.calcite.SysLimit; +import org.opensearch.sql.calcite.utils.CalciteToolsHelper.OpenSearchRelRunners; +import org.opensearch.sql.executor.QueryType; + +/** Base class for integration test based on RelNode tree. Mainly for testing internal functions */ +public abstract class CalcitePPLRelNodeIntegTestCase extends CalcitePPLIntegTestCase { + TestContext context; + + @Override + public void init() throws IOException { + super.init(); + context = createTestContext(); + enableCalcite(); + } + + protected static class TestContext { + final CalcitePlanContext planContext; + final RelBuilder relBuilder; + final RexBuilder rexBuilder; + + TestContext(CalcitePlanContext planContext, RelBuilder relBuilder, RexBuilder rexBuilder) { + this.planContext = planContext; + this.relBuilder = relBuilder; + this.rexBuilder = rexBuilder; + } + } + + @FunctionalInterface + protected interface ResultVerifier { + void verify(ResultSet resultSet) throws SQLException; + } + + protected TestContext createTestContext() { + CalcitePlanContext planContext = createCalcitePlanContext(); + return new TestContext(planContext, planContext.relBuilder, planContext.rexBuilder); + } + + protected RelDataType createMapType(RexBuilder rexBuilder) { + RelDataType stringType = rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR); + RelDataType anyType = rexBuilder.getTypeFactory().createSqlType(SqlTypeName.ANY); + return rexBuilder.getTypeFactory().createMapType(stringType, anyType); + } + + protected RelDataType createStringArrayType(RexBuilder rexBuilder) { + RelDataType stringType = rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR); + return rexBuilder.getTypeFactory().createArrayType(stringType, -1); + } + + protected RexNode createStringArray(RexBuilder rexBuilder, String... values) { + RelDataType stringType = rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR); + RelDataType arrayType = rexBuilder.getTypeFactory().createArrayType(stringType, -1); + + List elements = new java.util.ArrayList<>(); + for (String value : values) { + elements.add(rexBuilder.makeLiteral(value)); + } + + return rexBuilder.makeCall(arrayType, SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, elements); + } + + protected void executeRelNodeAndVerify( + CalcitePlanContext planContext, RelNode relNode, ResultVerifier verifier) + throws SQLException { + try (PreparedStatement statement = OpenSearchRelRunners.run(planContext, relNode)) { + ResultSet resultSet = statement.executeQuery(); + verifier.verify(resultSet); + } + } + + protected void verifyColumns(ResultSet resultSet, String... expectedColumnNames) + throws SQLException { + assertEquals(expectedColumnNames.length, resultSet.getMetaData().getColumnCount()); + + for (int i = 0; i < expectedColumnNames.length; i++) { + String expectedName = expectedColumnNames[i]; + String actualName = resultSet.getMetaData().getColumnName(i + 1); + assertEquals(expectedName, actualName); + } + } + + protected CalcitePlanContext createCalcitePlanContext() { + // Create a Frameworks.ConfigBuilder similar to CalcitePPLAbstractTest + final SchemaPlus rootSchema = Frameworks.createRootSchema(true); + Frameworks.ConfigBuilder config = + Frameworks.newConfigBuilder() + .parserConfig(SqlParser.Config.DEFAULT) + .defaultSchema(rootSchema) + .traitDefs((List) null) + .programs(Programs.heuristicJoinOrder(Programs.RULE_SET, true, 2)); + + config.context(Contexts.of(RelBuilder.Config.DEFAULT)); + + return CalcitePlanContext.create(config.build(), SysLimit.DEFAULT, QueryType.PPL); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/JsonExtractAllFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/JsonExtractAllFunctionIT.java new file mode 100644 index 00000000000..44997e0538e --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/JsonExtractAllFunctionIT.java @@ -0,0 +1,290 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.standalone; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.type.SqlTypeName; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.PPLFuncImpTable; + +public class JsonExtractAllFunctionIT extends CalcitePPLRelNodeIntegTestCase { + + private static final String RESULT_FIELD = "result"; + private static final String ID_FIELD = "id"; + + @Test + public void testJsonExtractAllWithNullInput() throws Exception { + RelDataType stringType = context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR); + RexNode nullJson = context.rexBuilder.makeNullLiteral(stringType); + + RexNode jsonExtractAllCall = + PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, BuiltinFunctionName.JSON_EXTRACT_ALL, nullJson); + + RelNode relNode = + context + .relBuilder + .values(new String[] {ID_FIELD}, 1) + .project(context.relBuilder.alias(jsonExtractAllCall, RESULT_FIELD)) + .build(); + + executeRelNodeAndVerify( + context.planContext, + relNode, + resultSet -> { + assertTrue(resultSet.next()); + verifyColumns(resultSet, RESULT_FIELD); + assertNull(resultSet.getObject(1)); + }); + } + + @Test + public void testJsonExtractAllWithSimpleObject() throws Exception { + String jsonString = "{\"name\": \"John\", \"age\": 30}"; + RexNode jsonLiteral = context.rexBuilder.makeLiteral(jsonString); + + RexNode jsonExtractAllCall = + PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, BuiltinFunctionName.JSON_EXTRACT_ALL, jsonLiteral); + + RelNode relNode = + context + .relBuilder + .values(new String[] {ID_FIELD}, 1) + .project(context.relBuilder.alias(jsonExtractAllCall, RESULT_FIELD)) + .build(); + + executeRelNodeAndVerify( + context.planContext, + relNode, + resultSet -> { + assertTrue(resultSet.next()); + verifyColumns(resultSet, RESULT_FIELD); + + Map map = getMap(resultSet, 1); + assertEquals("John", map.get("name")); + assertEquals(30, map.get("age")); + assertEquals(2, map.size()); + }); + } + + private Map getMap(ResultSet resultSet, int columnIndex) throws SQLException { + Object result = resultSet.getObject(columnIndex); + assertNotNull(result); + assertTrue(result instanceof Map); + + @SuppressWarnings("unchecked") + Map map = (Map) result; + System.out.println( + "map: " + + map.entrySet().stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .collect(Collectors.joining(", "))); + return map; + } + + @Test + public void testJsonExtractAllWithNestedObject() throws Exception { + String jsonString = "{\"user\": {\"name\": \"John\", \"age\": 30}, \"active\": true}"; + RexNode jsonLiteral = context.rexBuilder.makeLiteral(jsonString); + + RexNode jsonExtractAllCall = + PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, BuiltinFunctionName.JSON_EXTRACT_ALL, jsonLiteral); + + RelNode relNode = + context + .relBuilder + .values(new String[] {ID_FIELD}, 1) + .project(context.relBuilder.alias(jsonExtractAllCall, RESULT_FIELD)) + .build(); + + executeRelNodeAndVerify( + context.planContext, + relNode, + resultSet -> { + assertTrue(resultSet.next()); + verifyColumns(resultSet, RESULT_FIELD); + + Map map = getMap(resultSet, 1); + assertEquals("John", map.get("user.name")); + assertEquals(30, map.get("user.age")); + assertEquals(true, map.get("active")); + assertEquals(3, map.size()); + }); + } + + @Test + public void testJsonExtractAllWithArray() throws Exception { + String jsonString = "{\"tags\": [\"java\", \"sql\", \"opensearch\"]}"; + RexNode jsonLiteral = context.rexBuilder.makeLiteral(jsonString); + + RexNode jsonExtractAllCall = + PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, BuiltinFunctionName.JSON_EXTRACT_ALL, jsonLiteral); + + RelNode relNode = + context + .relBuilder + .values(new String[] {ID_FIELD}, 1) + .project(context.relBuilder.alias(jsonExtractAllCall, RESULT_FIELD)) + .build(); + + executeRelNodeAndVerify( + context.planContext, + relNode, + resultSet -> { + assertTrue(resultSet.next()); + verifyColumns(resultSet, RESULT_FIELD); + + Map map = getMap(resultSet, 1); + List tags = getList(map, "tags{}"); + + assertEquals(3, tags.size()); + assertEquals("java", tags.get(0)); + assertEquals("sql", tags.get(1)); + assertEquals("opensearch", tags.get(2)); + }); + } + + @Test + public void testJsonExtractAllWithArrayOfObjects() throws Exception { + String jsonString = "{\"users\": [{\"name\": \"John\"}, {\"name\": \"Jane\"}]}"; + RexNode jsonLiteral = context.rexBuilder.makeLiteral(jsonString); + + RexNode jsonExtractAllCall = + PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, BuiltinFunctionName.JSON_EXTRACT_ALL, jsonLiteral); + + RelNode relNode = + context + .relBuilder + .values(new String[] {ID_FIELD}, 1) + .project(context.relBuilder.alias(jsonExtractAllCall, RESULT_FIELD)) + .build(); + + executeRelNodeAndVerify( + context.planContext, + relNode, + resultSet -> { + assertTrue(resultSet.next()); + verifyColumns(resultSet, RESULT_FIELD); + + Map map = getMap(resultSet, 1); + List names = getList(map, "users{}.name"); + assertEquals(2, names.size()); + assertEquals("John", names.get(0)); + assertEquals("Jane", names.get(1)); + assertEquals(1, map.size()); // Only flattened key should exist + }); + } + + @Test + public void testJsonExtractAllWithTopLevelArray() throws Exception { + String jsonString = "[{\"id\": 1}, {\"id\": 2}]"; + RexNode jsonLiteral = context.rexBuilder.makeLiteral(jsonString); + + RexNode jsonExtractAllCall = + PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, BuiltinFunctionName.JSON_EXTRACT_ALL, jsonLiteral); + + RelNode relNode = + context + .relBuilder + .values(new String[] {ID_FIELD}, 1) + .project(context.relBuilder.alias(jsonExtractAllCall, RESULT_FIELD)) + .build(); + + executeRelNodeAndVerify( + context.planContext, + relNode, + resultSet -> { + assertTrue(resultSet.next()); + verifyColumns(resultSet, RESULT_FIELD); + + Map map = getMap(resultSet, 1); + List ids = getList(map, "{}.id"); + assertEquals(2, ids.size()); + assertEquals(1, ids.get(0)); + assertEquals(2, ids.get(1)); + assertEquals(1, map.size()); + }); + } + + @SuppressWarnings("unchecked") + private List getList(Map map, String key) { + Object value = map.get(key); + assertNotNull(value); + assertTrue(value instanceof List); + + return (List) value; + } + + @Test + public void testJsonExtractAllWithEmptyObject() throws Exception { + String jsonString = "{}"; + RexNode jsonLiteral = context.rexBuilder.makeLiteral(jsonString); + + RexNode jsonExtractAllCall = + PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, BuiltinFunctionName.JSON_EXTRACT_ALL, jsonLiteral); + + RelNode relNode = + context + .relBuilder + .values(new String[] {ID_FIELD}, 1) + .project(context.relBuilder.alias(jsonExtractAllCall, RESULT_FIELD)) + .build(); + + executeRelNodeAndVerify( + context.planContext, + relNode, + resultSet -> { + assertTrue(resultSet.next()); + verifyColumns(resultSet, RESULT_FIELD); + + Map map = getMap(resultSet, 1); + assertTrue(map.isEmpty()); + }); + } + + @Test + public void testJsonExtractAllWithInvalidJson() throws Exception { + String invalidJsonString = "{\"name\": \"John\", \"age\":}"; + RexNode jsonLiteral = context.rexBuilder.makeLiteral(invalidJsonString); + + RexNode jsonExtractAllCall = + PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, BuiltinFunctionName.JSON_EXTRACT_ALL, jsonLiteral); + + RelNode relNode = + context + .relBuilder + .values(new String[] {ID_FIELD}, 1) + .project(context.relBuilder.alias(jsonExtractAllCall, RESULT_FIELD)) + .build(); + + executeRelNodeAndVerify( + context.planContext, + relNode, + resultSet -> { + assertTrue(resultSet.next()); + verifyColumns(resultSet, RESULT_FIELD); + + Map map = getMap(resultSet, 1); + assertEquals("John", map.get("name")); + assertEquals(1, map.size()); + }); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/MapAppendFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/MapAppendFunctionIT.java new file mode 100644 index 00000000000..a77c3270e3b --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/MapAppendFunctionIT.java @@ -0,0 +1,158 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.standalone; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.PPLFuncImpTable; + +public class MapAppendFunctionIT extends CalcitePPLRelNodeIntegTestCase { + + private static final String MAP_FIELD = "map"; + private static final String ID_FIELD = "id"; + + @Test + public void testMapAppendWithNonOverlappingKeys() throws Exception { + RexNode map1 = createMap("key1", "value1", "key2", "value2"); + RexNode map2 = createMap("key3", "value3", "key4", "value4"); + RexNode mapAppendCall = + PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, BuiltinFunctionName.MAP_APPEND, map1, map2); + RelNode relNode = + context + .relBuilder + .values(new String[] {ID_FIELD}, 1) + .project(context.relBuilder.alias(mapAppendCall, MAP_FIELD)) + .build(); + + executeRelNodeAndVerify( + context.planContext, + relNode, + resultSet -> { + Map result = getResultMapField(resultSet); + assertEquals(4, result.size()); + assertMapListValue(result, "key1", "value1"); + assertMapListValue(result, "key2", "value2"); + assertMapListValue(result, "key3", "value3"); + assertMapListValue(result, "key4", "value4"); + }); + } + + @Test + public void testMapAppendWithOverlappingKeys() throws Exception { + RexNode map1 = createMap("key1", "value1", "key2", "value2"); + RexNode map2 = createMap("key2", "value3", "key3", "value4"); + RexNode mapAppendCall = + PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, BuiltinFunctionName.MAP_APPEND, map1, map2); + RelNode relNode = + context + .relBuilder + .values(new String[] {ID_FIELD}, 1) + .project(context.relBuilder.alias(mapAppendCall, MAP_FIELD)) + .build(); + + executeRelNodeAndVerify( + context.planContext, + relNode, + resultSet -> { + Map result = getResultMapField(resultSet); + assertEquals(3, result.size()); + assertMapListValue(result, "key1", "value1"); + assertMapListValue(result, "key2", "value2", "value3"); + assertMapListValue(result, "key3", "value4"); + }); + } + + @Test + public void testMapAppendWithSingleNull(RexNode map1, RexNode map2) throws Exception { + RelDataType mapType = createMapType(context.rexBuilder); + RexNode nullMap = context.rexBuilder.makeNullLiteral(mapType); + RexNode map = createMap("key1", "value1"); + testWithSingleNull(map, nullMap); + testWithSingleNull(nullMap, map); + } + + private void testWithSingleNull(RexNode map1, RexNode map2) throws Exception { + RexNode mapAppendCall = + PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, BuiltinFunctionName.MAP_APPEND, map1, map2); + RelNode relNode = + context + .relBuilder + .values(new String[] {ID_FIELD}, 1) + .project(context.relBuilder.alias(mapAppendCall, MAP_FIELD)) + .build(); + + executeRelNodeAndVerify( + context.planContext, + relNode, + resultSet -> { + Map result = getResultMapField(resultSet); + assertEquals(1, result.size()); + assertMapListValue(result, "key1", "value1"); + }); + } + + @Test + public void testMapAppendWithNullMaps() throws Exception { + RelDataType mapType = createMapType(context.rexBuilder); + RexNode nullMap = context.rexBuilder.makeNullLiteral(mapType); + RexNode mapAppendCall = + PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, BuiltinFunctionName.MAP_APPEND, nullMap, nullMap); + RelNode relNode = + context + .relBuilder + .values(new String[] {ID_FIELD}, 1) + .project(context.relBuilder.alias(mapAppendCall, MAP_FIELD)) + .build(); + + executeRelNodeAndVerify( + context.planContext, + relNode, + resultSet -> { + assertNull(getResultMapField(resultSet)); + }); + } + + private RexNode createMap(String... keyValuePairs) { + RexNode[] args = new RexNode[keyValuePairs.length]; + for (int i = 0; i < keyValuePairs.length; i++) { + args[i] = context.rexBuilder.makeLiteral(keyValuePairs[i]); + } + + return context.rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, args); + } + + @SuppressWarnings("unchecked") + private Map getResultMapField(ResultSet resultSet) throws SQLException { + assertTrue(resultSet.next()); + verifyColumns(resultSet, MAP_FIELD); + Map result = (Map) resultSet.getObject(1); + return result; + } + + @SuppressWarnings("unchecked") + private void assertMapListValue(Map map, String key, Object... expectedValues) { + map.containsKey(key); + Object value = map.get(key); + assertTrue(value instanceof List); + List list = (List) value; + assertEquals(expectedValues.length, list.size()); + for (int i = 0; i < expectedValues.length; i++) { + assertEquals(expectedValues[i], list.get(i)); + } + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/MapConcatFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/MapConcatFunctionIT.java new file mode 100644 index 00000000000..5a7f6e18444 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/MapConcatFunctionIT.java @@ -0,0 +1,89 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.standalone; + +import java.util.Map; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.PPLFuncImpTable; + +public class MapConcatFunctionIT extends CalcitePPLRelNodeIntegTestCase { + + private static final String MAP_FIELD = "map"; + private static final String ID_FIELD = "id"; + + @Test + public void testMapConcatWithNullValues() throws Exception { + RelDataType mapType = createMapType(context.rexBuilder); + RexNode map1 = context.rexBuilder.makeNullLiteral(mapType); + RexNode map2 = context.rexBuilder.makeNullLiteral(mapType); + + RexNode mapConcatCall = + PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, BuiltinFunctionName.MAP_CONCAT, map1, map2); + + RelNode relNode = + context + .relBuilder + .values(new String[] {ID_FIELD}, 1) + .project(context.relBuilder.alias(mapConcatCall, MAP_FIELD)) + .build(); + + executeRelNodeAndVerify( + context.planContext, + relNode, + resultSet -> { + assertTrue(resultSet.next()); + verifyColumns(resultSet, MAP_FIELD); + assertNull(resultSet.getObject(1)); + }); + } + + @Test + public void testMapConcat() throws Exception { + RexNode map1 = + context.rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + context.rexBuilder.makeLiteral("key1"), + context.rexBuilder.makeLiteral("value1"), + context.rexBuilder.makeLiteral("key2"), + context.rexBuilder.makeLiteral("value2")); + RexNode map2 = + context.rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + context.rexBuilder.makeLiteral("key2"), + context.rexBuilder.makeLiteral("updated_value2"), + context.rexBuilder.makeLiteral("key3"), + context.rexBuilder.makeLiteral("value3")); + + RexNode mapConcatCall = + PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, BuiltinFunctionName.MAP_CONCAT, map1, map2); + + RelNode relNode = + context + .relBuilder + .values(new String[] {ID_FIELD}, 1) + .project(context.relBuilder.alias(mapConcatCall, MAP_FIELD)) + .build(); + + executeRelNodeAndVerify( + context.planContext, + relNode, + resultSet -> { + assertTrue(resultSet.next()); + verifyColumns(resultSet, MAP_FIELD); + Map result = (Map) resultSet.getObject(1); + assertEquals("value1", result.get("key1")); + assertEquals("updated_value2", result.get("key2")); + assertEquals("value3", result.get("key3")); + }); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/MapRemoveFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/MapRemoveFunctionIT.java new file mode 100644 index 00000000000..eeb006491b7 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/MapRemoveFunctionIT.java @@ -0,0 +1,173 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.standalone; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Map; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.PPLFuncImpTable; + +public class MapRemoveFunctionIT extends CalcitePPLRelNodeIntegTestCase { + + private static final String MAP_FIELD = "map"; + private static final String ID_FIELD = "id"; + + @Test + public void testMapRemoveWithNullMap() throws Exception { + RelDataType mapType = createMapType(context.rexBuilder); + RexNode nullMap = context.rexBuilder.makeNullLiteral(mapType); + RexNode keysArray = createStringArray(context.rexBuilder, "key1", "key2"); + RexNode mapRemoveCall = + PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, BuiltinFunctionName.MAP_REMOVE, nullMap, keysArray); + RelNode relNode = + context + .relBuilder + .values(new String[] {ID_FIELD}, 1) + .project(context.relBuilder.alias(mapRemoveCall, MAP_FIELD)) + .build(); + + executeRelNodeAndVerify( + context.planContext, + relNode, + resultSet -> { + assertTrue(resultSet.next()); + verifyColumns(resultSet, MAP_FIELD); + assertNull(resultSet.getObject(1)); + }); + } + + @Test + public void testMapRemoveWithNullKeys() throws Exception { + RexNode map = getBaseMap(); + RelDataType arrayType = createStringArrayType(context.rexBuilder); + RexNode nullKeys = context.rexBuilder.makeNullLiteral(arrayType); + RexNode mapRemoveCall = + PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, BuiltinFunctionName.MAP_REMOVE, map, nullKeys); + RelNode relNode = + context + .relBuilder + .values(new String[] {ID_FIELD}, 1) + .project(context.relBuilder.alias(mapRemoveCall, MAP_FIELD)) + .build(); + + executeRelNodeAndVerify( + context.planContext, + relNode, + resultSet -> { + Map result = getResultMapField(resultSet); + assertEquals(3, result.size()); + assertEquals("value1", result.get("key1")); + assertEquals("value2", result.get("key2")); + assertEquals("value3", result.get("key3")); + }); + } + + @Test + public void testMapRemoveExistingKeys() throws Exception { + RexNode map = getBaseMap(); + RexNode keysArray = createStringArray(context.rexBuilder, "key1", "key3"); + RexNode mapRemoveCall = + PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, BuiltinFunctionName.MAP_REMOVE, map, keysArray); + RelNode relNode = + context + .relBuilder + .values(new String[] {ID_FIELD}, 1) + .project(context.relBuilder.alias(mapRemoveCall, MAP_FIELD)) + .build(); + + executeRelNodeAndVerify( + context.planContext, + relNode, + resultSet -> { + Map result = getResultMapField(resultSet); + assertEquals(1, result.size()); + assertEquals("value2", result.get("key2")); + assertNull(result.get("key1")); + assertNull(result.get("key3")); + }); + } + + @Test + public void testMapRemoveNonExistingKeys() throws Exception { + RexNode map = getBaseMap(); + RexNode keysArray = createStringArray(context.rexBuilder, "key4", "key5"); + RexNode mapRemoveCall = + PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, BuiltinFunctionName.MAP_REMOVE, map, keysArray); + RelNode relNode = + context + .relBuilder + .values(new String[] {ID_FIELD}, 1) + .project(context.relBuilder.alias(mapRemoveCall, MAP_FIELD)) + .build(); + + executeRelNodeAndVerify( + context.planContext, + relNode, + resultSet -> { + Map result = getResultMapField(resultSet); + assertEquals(3, result.size()); + assertEquals("value1", result.get("key1")); + assertEquals("value2", result.get("key2")); + assertEquals("value3", result.get("key3")); + }); + } + + @Test + public void testMapRemoveEmptyKeysArray() throws Exception { + RexNode map = getBaseMap(); + RelDataType arrayType = createStringArrayType(context.rexBuilder); + RexNode nullKeys = context.rexBuilder.makeNullLiteral(arrayType); + RexNode mapRemoveCall = + PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, BuiltinFunctionName.MAP_REMOVE, map, nullKeys); + + RelNode relNode = + context + .relBuilder + .values(new String[] {ID_FIELD}, 1) + .project(context.relBuilder.alias(mapRemoveCall, MAP_FIELD)) + .build(); + + executeRelNodeAndVerify( + context.planContext, + relNode, + resultSet -> { + Map result = getResultMapField(resultSet); + assertEquals(3, result.size()); + assertEquals("value1", result.get("key1")); + assertEquals("value2", result.get("key2")); + assertEquals("value3", result.get("key3")); + }); + } + + private RexNode getBaseMap() { + return context.rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + context.rexBuilder.makeLiteral("key1"), + context.rexBuilder.makeLiteral("value1"), + context.rexBuilder.makeLiteral("key2"), + context.rexBuilder.makeLiteral("value2"), + context.rexBuilder.makeLiteral("key3"), + context.rexBuilder.makeLiteral("value3")); + } + + private Map getResultMapField(ResultSet resultSet) throws SQLException { + assertTrue(resultSet.next()); + verifyColumns(resultSet, MAP_FIELD); + Map result = (Map) resultSet.getObject(1); + return result; + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index 4e258088b5e..0ccf31f0998 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -51,6 +51,8 @@ public abstract class SQLIntegTestCase extends OpenSearchSQLRestTestCase { public static final String TRANSIENT = "transient"; public static final Integer DEFAULT_QUERY_SIZE_LIMIT = Integer.parseInt(System.getProperty("defaultQuerySizeLimit", "200")); + public static final Integer DEFAULT_QUERY_BUCKET_SIZE = + Integer.parseInt(System.getProperty("defaultQueryBucketSize", "1000")); public static final Integer DEFAULT_MAX_RESULT_WINDOW = Integer.parseInt(System.getProperty("defaultMaxResultWindow", "10000")); @@ -129,8 +131,8 @@ public static void dumpCoverage() { @AfterClass public static void cleanUpIndices() throws IOException { if (System.getProperty("tests.rest.bwcsuite") == null) { - wipeAllOpenSearchIndices(); - wipeAllClusterSettings(); +// wipeAllOpenSearchIndices(); +// wipeAllClusterSettings(); } } @@ -148,6 +150,20 @@ protected void resetQuerySizeLimit() throws IOException { DEFAULT_QUERY_SIZE_LIMIT.toString())); } + protected void setQueryBucketSize(Integer limit) throws IOException { + updateClusterSettings( + new ClusterSetting( + "transient", Settings.Key.QUERY_BUCKET_SIZE.getKeyValue(), limit.toString())); + } + + protected void resetQueryBucketSize() throws IOException { + updateClusterSettings( + new ClusterSetting( + "transient", + Settings.Key.QUERY_BUCKET_SIZE.getKeyValue(), + DEFAULT_QUERY_BUCKET_SIZE.toString())); + } + @SneakyThrows protected void setDataSourcesEnabled(String clusterSettingType, boolean value) { updateClusterSettings( @@ -565,6 +581,11 @@ public enum Index { "location2", getLocationIndexMapping(), "src/test/resources/locations2.json"), + LOCATIONS_TYPE_CONFLICT( + TestsConstants.TEST_INDEX_LOCATIONS_TYPE_CONFLICT, + "locations", + getLocationsTypeConflictIndexMapping(), + "src/test/resources/locations_type_conflict.json"), NESTED( TestsConstants.TEST_INDEX_NESTED_TYPE, "nestedType", @@ -635,6 +656,11 @@ public enum Index { "_doc", getOrderIndexMapping(), "src/test/resources/order.json"), + TIME_TEST_DATA2( + "opensearch-sql_test_index_time_data2", + "time_data", + getMappingFile("time_test_data_index_mapping.json"), + "src/test/resources/time_test_data2.json"), WEBLOG( TestsConstants.TEST_INDEX_WEBLOGS, "weblogs", @@ -896,7 +922,12 @@ public enum Index { "events_null", "events_null", "{\"mappings\":{\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"host\":{\"type\":\"text\"},\"cpu_usage\":{\"type\":\"double\"},\"region\":{\"type\":\"keyword\"}}}}", - "src/test/resources/events_null.json"); + "src/test/resources/events_null.json"), + EVENTS_TRAFFIC( + "events_traffic", + "events_traffic", + getMappingFile("events_traffic_index_mapping.json"), + "src/test/resources/events_traffic.json"); private final String name; private final String type; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index 09a24269692..a94e89ec0e6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -170,6 +170,11 @@ public static String getLocationIndexMapping() { return getMappingFile(mappingFile); } + public static String getLocationsTypeConflictIndexMapping() { + String mappingFile = "locations_type_conflict_index_mapping.json"; + return getMappingFile(mappingFile); + } + public static String getEmployeeNestedTypeIndexMapping() { String mappingFile = "employee_nested_type_index_mapping.json"; return getMappingFile(mappingFile); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index 17e30e5938c..76923dbd984 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -27,6 +27,8 @@ public class TestsConstants { public static final String TEST_INDEX_ODBC = TEST_INDEX + "_odbc"; public static final String TEST_INDEX_LOCATION = TEST_INDEX + "_location"; public static final String TEST_INDEX_LOCATION2 = TEST_INDEX + "_location2"; + public static final String TEST_INDEX_LOCATIONS_TYPE_CONFLICT = + TEST_INDEX + "_locations_type_conflict"; public static final String TEST_INDEX_NESTED_TYPE = TEST_INDEX + "_nested_type"; public static final String TEST_INDEX_NESTED_TYPE_WITHOUT_ARRAYS = TEST_INDEX + "_nested_type_without_arrays"; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java index adc160748e7..2319cf703a4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java @@ -12,7 +12,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_TIME_DATA; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOGS; import static org.opensearch.sql.util.MatcherUtils.assertJsonEqualsIgnoreId; -import static org.opensearch.sql.util.MatcherUtils.assertYamlEqualsJsonIgnoreId; +import static org.opensearch.sql.util.MatcherUtils.assertYamlEqualsIgnoreId; import java.io.IOException; import java.util.Locale; @@ -38,9 +38,9 @@ public void init() throws Exception { @Test public void testExplain() throws IOException { String expected = loadExpectedPlan("explain_output.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account" + "| where age > 30 " + "| stats avg(age) AS avg_age by state, city " @@ -54,9 +54,9 @@ public void testExplain() throws IOException { @Test public void testFilterPushDownExplain() throws IOException { String expected = loadExpectedPlan("explain_filter_push.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account" + "| where age > 30 " + "| where age < 40 " @@ -67,9 +67,9 @@ public void testFilterPushDownExplain() throws IOException { @Test public void testFilterByCompareStringTimestampPushDownExplain() throws IOException { String expected = loadExpectedPlan("explain_filter_push_compare_timestamp_string.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_bank" + "| where birthdate > '2016-12-08 00:00:00.000000000' " + "| where birthdate < '2018-11-09 00:00:00.000000000' ")); @@ -78,9 +78,9 @@ public void testFilterByCompareStringTimestampPushDownExplain() throws IOExcepti @Test public void testFilterByCompareStringDatePushDownExplain() throws IOException { String expected = loadExpectedPlan("explain_filter_push_compare_date_string.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_date_formats | fields yyyy-MM-dd" + "| where yyyy-MM-dd > '2016-12-08 00:00:00.123456789' " + "| where yyyy-MM-dd < '2018-11-09 00:00:00.000000000' ")); @@ -89,9 +89,9 @@ public void testFilterByCompareStringDatePushDownExplain() throws IOException { @Test public void testFilterByCompareStringTimePushDownExplain() throws IOException { String expected = loadExpectedPlan("explain_filter_push_compare_time_string.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_date_formats | fields custom_time" + "| where custom_time > '2016-12-08 12:00:00.123456789' " + "| where custom_time < '2018-11-09 19:00:00.123456789' ")); @@ -141,9 +141,9 @@ public void testWeekArgumentCoercion() throws IOException { @Test public void testFilterAndAggPushDownExplain() throws IOException { String expected = loadExpectedPlan("explain_filter_agg_push.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account" + "| where age > 30 " + "| stats avg(age) AS avg_age by state, city")); @@ -172,9 +172,9 @@ public void testSortPushDownExplain() throws IOException { @Test public void testSortWithCountPushDownExplain() throws IOException { String expected = loadExpectedPlan("explain_sort_count_push.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString("source=opensearch-sql_test_index_account | sort 5 age | fields age")); + explainQueryYaml("source=opensearch-sql_test_index_account | sort 5 age | fields age")); } @Test @@ -183,7 +183,7 @@ public void testSortWithDescPushDownExplain() throws IOException { assertJsonEqualsIgnoreId( expected, explainQueryToString( - "source=opensearch-sql_test_index_account | sort age, - firstname desc | fields age," + "source=opensearch-sql_test_index_account | sort age desc, firstname | fields age," + " firstname")); } @@ -256,27 +256,21 @@ public void testSortWithRenameExplain() throws IOException { @Test public void testSortThenLimitExplain() throws IOException { String expected = loadExpectedPlan("explain_sort_then_limit_push.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account" + "| sort age " + "| head 5 " + "| fields age")); } - /** - * Push down LIMIT only Sort should NOT be pushed down since DSL process limit before sort when - * they coexist - */ @Test public void testLimitThenSortExplain() throws IOException { - // TODO: Fix the expected output in expectedOutput/ppl/explain_limit_then_sort_push.json (v2) - // limit-then-sort should not be pushed down. String expected = loadExpectedPlan("explain_limit_then_sort_push.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account" + "| head 5 " + "| sort age " @@ -286,9 +280,9 @@ public void testLimitThenSortExplain() throws IOException { @Test public void testLimitPushDownExplain() throws IOException { String expected = loadExpectedPlan("explain_limit_push.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account" + "| eval ageMinus = age - 30 " + "| head 5 " @@ -298,9 +292,9 @@ public void testLimitPushDownExplain() throws IOException { @Test public void testLimitWithFilterPushdownExplain() throws IOException { String expectedFilterThenLimit = loadExpectedPlan("explain_filter_then_limit_push.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expectedFilterThenLimit, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account" + "| where age > 30 " + "| head 5 " @@ -309,9 +303,9 @@ public void testLimitWithFilterPushdownExplain() throws IOException { // The filter in limit-then-filter queries should not be pushed since the current DSL will // execute it as filter-then-limit String expectedLimitThenFilter = loadExpectedPlan("explain_limit_then_filter_push.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expectedLimitThenFilter, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account" + "| head 5 " + "| where age > 30 " @@ -321,27 +315,27 @@ public void testLimitWithFilterPushdownExplain() throws IOException { @Test public void testMultipleLimitExplain() throws IOException { String expected5Then10 = loadExpectedPlan("explain_limit_5_10_push.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected5Then10, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account" + "| head 5 " + "| head 10 " + "| fields age")); String expected10Then5 = loadExpectedPlan("explain_limit_10_5_push.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected10Then5, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account" + "| head 10 " + "| head 5 " + "| fields age")); String expected10from1then10from2 = loadExpectedPlan("explain_limit_10from1_10from2_push.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected10from1then10from2, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account" + "| head 10 from 1 " + "| head 10 from 2 " @@ -349,9 +343,9 @@ public void testMultipleLimitExplain() throws IOException { // The second limit should not be pushed down for limit-filter-limit queries String expected10ThenFilterThen5 = loadExpectedPlan("explain_limit_10_filter_5_push.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected10ThenFilterThen5, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account" + "| head 10 " + "| where age > 30 " @@ -362,9 +356,9 @@ public void testMultipleLimitExplain() throws IOException { @Test public void testLimitWithMultipleOffsetPushdownExplain() throws IOException { String expected = loadExpectedPlan("explain_limit_offsets_push.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account" + "| head 10 from 1 " + "| head 5 from 2 " @@ -384,9 +378,9 @@ public void testFillNullPushDownExplain() throws IOException { @Test public void testTrendlinePushDownExplain() throws IOException { String expected = loadExpectedPlan("explain_trendline_push.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account" + "| head 5 " + "| trendline sma(2, age) as ageTrend " @@ -397,9 +391,9 @@ public void testTrendlinePushDownExplain() throws IOException { public void testTrendlineWithSortPushDownExplain() throws IOException { String expected = loadExpectedPlan("explain_trendline_sort_push.yaml"); // Sort will not be pushed down because there's a head before it. - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account" + "| head 5 " + "| trendline sort age sma(2, age) as ageTrend " @@ -422,17 +416,16 @@ public void testExplainModeUnsupportedInV2() throws IOException { public void testPatternsSimplePatternMethodWithoutAggExplain() throws IOException { // TODO: Correct calcite expected result once pushdown is supported String expected = loadExpectedPlan("explain_patterns_simple_pattern.yaml"); - assertYamlEqualsJsonIgnoreId( - expected, - explainQueryToString("source=opensearch-sql_test_index_account | patterns email")); + assertYamlEqualsIgnoreId( + expected, explainQueryYaml("source=opensearch-sql_test_index_account | patterns email")); } @Test public void testPatternsSimplePatternMethodWithAggPushDownExplain() throws IOException { String expected = loadExpectedPlan("explain_patterns_simple_pattern_agg_push.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | patterns email mode=aggregation" + " show_numbered_token=true")); } @@ -441,9 +434,9 @@ public void testPatternsSimplePatternMethodWithAggPushDownExplain() throws IOExc public void testPatternsBrainMethodWithAggPushDownExplain() throws IOException { // TODO: Correct calcite expected result once pushdown is supported String expected = loadExpectedPlan("explain_patterns_brain_agg_push.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account" + "| patterns email method=brain mode=aggregation show_numbered_token=true")); } @@ -554,29 +547,29 @@ public void testMultiFieldsRelevanceQueryFunctionExplain() throws IOException { @Test public void testKeywordLikeFunctionExplain() throws IOException { - String expected = loadExpectedPlan("explain_keyword_like_function.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_keyword_like_function.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | where like(firstname, '%mbe%')")); } @Test public void testTextLikeFunctionExplain() throws IOException { - String expected = loadExpectedPlan("explain_text_like_function.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_text_like_function.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | where like(address, '%Holmes%')")); } @Ignore("The serialized string is unstable because of function properties") @Test public void testFilterScriptPushDownExplain() throws Exception { - String expected = loadExpectedPlan("explain_filter_script_push.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_filter_script_push.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | where firstname ='Amber' and age - 2 = 30 |" + " fields firstname, age")); } @@ -584,10 +577,10 @@ public void testFilterScriptPushDownExplain() throws Exception { @Ignore("The serialized string is unstable because of function properties") @Test public void testFilterFunctionScriptPushDownExplain() throws Exception { - String expected = loadExpectedPlan("explain_filter_function_script_push.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_filter_function_script_push.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | where length(firstname) = 5 and abs(age) =" + " 32 and balance = 39225 | fields firstname, age")); } @@ -609,10 +602,10 @@ public void testDifferentFilterScriptPushDownBehaviorExplain() throws Exception @Test public void testExplainOnTake() throws IOException { - String expected = loadExpectedPlan("explain_take.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_take.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | stats take(firstname, 2) as take")); } @@ -628,10 +621,10 @@ public void testExplainOnPercentile() throws IOException { @Test public void testExplainOnAggregationWithFunction() throws IOException { - String expected = loadExpectedPlan("explain_agg_with_script.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_agg_with_script.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format( "source=%s | eval len = length(gender) | stats sum(balance + 100) as sum by len," + " gender ", @@ -641,9 +634,9 @@ public void testExplainOnAggregationWithFunction() throws IOException { @Test public void testSearchCommandWithAbsoluteTimeRange() throws IOException { String expected = loadExpectedPlan("search_with_absolute_time_range.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format( "source=%s earliest='2022-12-10 13:11:04' latest='2025-09-03 15:10:00'", TEST_INDEX_TIME_DATA))); @@ -651,32 +644,32 @@ public void testSearchCommandWithAbsoluteTimeRange() throws IOException { @Test public void testSearchCommandWithRelativeTimeRange() throws IOException { - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( loadExpectedPlan("search_with_relative_time_range.yaml"), // "", - explainQueryToString( + explainQueryYaml( String.format("source=%s earliest=-1q latest=+30d", TEST_INDEX_TIME_DATA))); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( loadExpectedPlan("search_with_relative_time_snap.yaml"), - explainQueryToString( + explainQueryYaml( String.format("source=%s earliest='-1q@year' latest=now", TEST_INDEX_TIME_DATA))); } @Test public void testSearchCommandWithNumericTimeRange() throws IOException { String expected = loadExpectedPlan("search_with_numeric_time_range.yaml"); - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format("source=%s earliest=1 latest=1754020061.123456", TEST_INDEX_TIME_DATA))); } @Test public void testSearchCommandWithChainedTimeModifier() throws IOException { - assertYamlEqualsJsonIgnoreId( + assertYamlEqualsIgnoreId( loadExpectedPlan("search_with_chained_time_modifier.yaml"), - explainQueryToString( + explainQueryYaml( String.format( "source=%s earliest='-3d@d-2h+10m' latest='-1d+1y@mon'", TEST_INDEX_TIME_DATA))); } @@ -712,17 +705,14 @@ public void testExplainSearchWildcardStar() throws IOException { String.format("search source=%s severityText=ERR*", TEST_INDEX_OTEL_LOGS))); } - protected String loadExpectedPlan(String fileName) throws IOException { - String prefix; - if (isCalciteEnabled()) { - if (isPushdownDisabled()) { - prefix = "expectedOutput/calcite_no_pushdown/"; - } else { - prefix = "expectedOutput/calcite/"; - } - } else { - prefix = "expectedOutput/ppl/"; - } - return loadFromFile(prefix + fileName); + @Test + public void testStatsByDependentGroupFieldsExplain() throws IOException { + String expected = loadExpectedPlan("explain_agg_group_merge.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + "source=opensearch-sql_test_index_account" + + "| eval age1 = age * 10, age2 = age + 10, age3 = 10" + + "| stats count() by age1, age2, age3, age")); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/GeoIpFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/GeoIpFunctionsIT.java index fe747fcdc03..14c6e2fb8f8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/GeoIpFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/GeoIpFunctionsIT.java @@ -40,7 +40,7 @@ public class GeoIpFunctionsIT extends PPLIntegTestCase { "endpoint", "https://raw.githubusercontent.com/opensearch-project/geospatial/main/src/test/resources/ip2geo/server/city/manifest.json"); - private static String DATASOURCE_NAME = "dummycityindex"; + protected static String DATASOURCE_NAME = "dummycityindex"; private static String PLUGIN_NAME = "opensearch-geospatial"; @@ -83,7 +83,7 @@ public void testGeoIpEnrichment() { String.format( "search source=%s | eval enrichmentResult = geoip(\\\"%s\\\",%s) | fields name, ip," + " enrichmentResult", - TEST_INDEX_GEOIP, "dummycityindex", "ip")); + TEST_INDEX_GEOIP, DATASOURCE_NAME, "ip")); verifyColumn(resultGeoIp, columnName("name"), columnName("ip"), columnName("enrichmentResult")); verifyDataRows( @@ -101,7 +101,7 @@ public void testGeoIpEnrichmentWithSingleOption() { String.format( "search source=%s | eval enrichmentResult = geoip(\\\"%s\\\",%s,\\\"%s\\\") |" + " fields name, ip, enrichmentResult", - TEST_INDEX_GEOIP, "dummycityindex", "ip", "city")); + TEST_INDEX_GEOIP, DATASOURCE_NAME, "ip", "city")); verifyColumn(resultGeoIp, columnName("name"), columnName("ip"), columnName("enrichmentResult")); verifyDataRows( @@ -119,7 +119,7 @@ public void testGeoIpEnrichmentWithSpaceSeparatedMultipleOptions() { String.format( "search source=%s | eval enrichmentResult = geoip(\\\"%s\\\",%s,\\\"%s\\\") |" + " fields name, ip, enrichmentResult", - TEST_INDEX_GEOIP, "dummycityindex", "ip", "city , country")); + TEST_INDEX_GEOIP, DATASOURCE_NAME, "ip", "city , country")); verifyColumn(resultGeoIp, columnName("name"), columnName("ip"), columnName("enrichmentResult")); verifyDataRows( diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLConcurrencyIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLConcurrencyIT.java new file mode 100644 index 00000000000..33798de61b3 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLConcurrencyIT.java @@ -0,0 +1,118 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.sql.legacy.TestsConstants; + +/** Verifies PPL aggregations remain stable when executed concurrently. */ +public class PPLConcurrencyIT extends PPLIntegTestCase { + + private static final int THREAD_POOL_SIZE = 256; + private static final long QUERY_TIMEOUT_SECONDS = 60; + + @Override + public void init() throws Exception { + super.init(); + loadIndex(Index.ACCOUNT); + } + + @Test + public void aggregationsHandleConcurrentPplQueries() throws Exception { + String countQuery = + String.format("source=%s | stats count(age) as cnt", TestsConstants.TEST_INDEX_ACCOUNT); + String sumQuery = + String.format( + "source=%s | stats sum(balance) as total_sales", TestsConstants.TEST_INDEX_ACCOUNT); + + long expectedCount = extractNumber(executeQuery(countQuery)).longValue(); + double expectedSum = extractNumber(executeQuery(sumQuery)).doubleValue(); + + runConcurrentRound(countQuery, expectedCount, sumQuery, expectedSum); + } + + private void runConcurrentRound( + String countQuery, long expectedCount, String sumQuery, double expectedSum) throws Exception { + ExecutorService executor = Executors.newFixedThreadPool(THREAD_POOL_SIZE); + List> tasks = new ArrayList<>(); + + for (int i = 0; i < THREAD_POOL_SIZE; i++) { + tasks.add(countTask(countQuery, expectedCount)); + tasks.add(sumTask(sumQuery, expectedSum)); + } + + Collections.shuffle(tasks, ThreadLocalRandom.current()); + + List> futures = new ArrayList<>(); + try { + for (Callable task : tasks) { + futures.add(executor.submit(task)); + } + waitForTasks(futures); + } finally { + executor.shutdown(); + if (!executor.awaitTermination(QUERY_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + executor.shutdownNow(); + } + } + } + + private Callable countTask(String query, long expected) { + return () -> { + long actual = extractNumber(executeQuerySafely(query)).longValue(); + assertEquals("Unexpected COUNT result", expected, actual); + return null; + }; + } + + private Callable sumTask(String query, double expected) { + return () -> { + double actual = extractNumber(executeQuerySafely(query)).doubleValue(); + assertEquals("Unexpected SUM result", expected, actual, 1e-6); + return null; + }; + } + + private void waitForTasks(List> tasks) throws Exception { + for (Future task : tasks) { + task.get(QUERY_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + tasks.clear(); + } + + private JSONObject executeQuerySafely(String query) { + try { + return executeQuery(query); + } catch (IOException e) { + throw new RuntimeException("Failed to execute PPL query: " + query, e); + } + } + + private Number extractNumber(JSONObject response) { + JSONArray rows = response.getJSONArray("datarows"); + assertEquals("Expected a single row", 1, rows.length()); + JSONArray row = rows.getJSONArray(0); + assertEquals("Expected a single column", 1, row.length()); + Object value = row.get(0); + assertTrue("Expected numeric result but got " + value, value instanceof Number); + return (Number) value; + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java index 0ac42564192..16dd4e541d0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java @@ -31,14 +31,16 @@ import org.opensearch.sql.common.setting.Settings.Key; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.util.RetryProcessor; -import org.opensearch.sql.utils.YamlFormatter; /** OpenSearch Rest integration test base for PPL testing. */ public abstract class PPLIntegTestCase extends SQLIntegTestCase { private static final String EXTENDED_EXPLAIN_API_ENDPOINT = "/_plugins/_ppl/_explain?format=extended"; + private static final String YAML_EXPLAIN_API_ENDPOINT = "/_plugins/_ppl/_explain?format=yaml"; private static final Logger LOG = LogManager.getLogger(); @Rule public final RetryProcessor retryProcessor = new RetryProcessor(); + public static final Integer DEFAULT_SUBSEARCH_MAXOUT = 10000; + public static final Integer DEFAULT_JOIN_SUBSEARCH_MAXOUT = 50000; @Override protected void init() throws Exception { @@ -48,7 +50,28 @@ protected void init() throws Exception { } protected JSONObject executeQuery(String query) throws IOException { - return jsonify(executeQueryToString(query)); + String text = executeQueryToString(query); + if (text == null) return null; + return jsonify(text); + } + + private String executeQuery(MapBuilder resultMap, Integer key, String ppl) throws IOException { + Response response; + try { + response = client().performRequest(buildRequest(ppl, QUERY_API_ENDPOINT)); + } catch (IOException e) { + resultMap.put(key, false); + return ""; + } + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode != 200) { + resultMap.put(key, false); + } else { + resultMap.put(key, true); + } + String responseBody = getResponseBody(response, true); + logger.info("Response {}", responseBody); + return responseBody; } protected String executeQueryToString(String query) throws IOException { @@ -61,10 +84,11 @@ protected String explainQueryToString(String query) throws IOException { return explainQueryToString(query, false); } - protected String explainQueryToYaml(String query) throws IOException { - String jsonResponse = explainQueryToString(query); - JSONObject jsonObject = jsonify(jsonResponse); - return YamlFormatter.formatToYaml(jsonObject); + protected String explainQueryYaml(String query) throws IOException { + Response response = client().performRequest(buildRequest(query, YAML_EXPLAIN_API_ENDPOINT)); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + String responseBody = getResponseBody(response, true); + return responseBody; } protected String explainQueryToString(String query, boolean extended) throws IOException { @@ -110,13 +134,22 @@ protected static String source(String index, String query) { protected void timing(MapBuilder builder, String query, String ppl) throws IOException { - executeQuery(ppl); // warm-up +// executeQuery(ppl); // warm-up long start = System.currentTimeMillis(); executeQuery(ppl); long duration = System.currentTimeMillis() - start; builder.put(query, duration); } + protected String runQuery(MapBuilder builder, MapBuilder resultMap, Integer queryNum, String ppl) + throws IOException { + long start = System.currentTimeMillis(); + String result = executeQuery(resultMap, queryNum, ppl); + long duration = System.currentTimeMillis() - start; + builder.put("q" + queryNum, duration); + return result; + } + protected void failWithMessage(String query, String message) { try { client().performRequest(buildRequest(query, QUERY_API_ENDPOINT)); @@ -334,6 +367,34 @@ public void updatePushdownSettings() throws IOException { } } + protected void setSubsearchMaxOut(Integer limit) throws IOException { + updateClusterSettings( + new SQLIntegTestCase.ClusterSetting( + "transient", Key.PPL_SUBSEARCH_MAXOUT.getKeyValue(), limit.toString())); + } + + protected void resetSubsearchMaxOut() throws IOException { + updateClusterSettings( + new SQLIntegTestCase.ClusterSetting( + "transient", + Settings.Key.PPL_SUBSEARCH_MAXOUT.getKeyValue(), + DEFAULT_SUBSEARCH_MAXOUT.toString())); + } + + protected void setJoinSubsearchMaxOut(Integer limit) throws IOException { + updateClusterSettings( + new SQLIntegTestCase.ClusterSetting( + "transient", Key.PPL_JOIN_SUBSEARCH_MAXOUT.getKeyValue(), limit.toString())); + } + + protected void resetJoinSubsearchMaxOut() throws IOException { + updateClusterSettings( + new SQLIntegTestCase.ClusterSetting( + "transient", + Settings.Key.PPL_JOIN_SUBSEARCH_MAXOUT.getKeyValue(), + DEFAULT_JOIN_SUBSEARCH_MAXOUT.toString())); + } + /** * Sanitizes the PPL query by removing block comments and replacing new lines with spaces. * @@ -361,4 +422,18 @@ protected static String loadFromFile(String filename) { throw new RuntimeException(e); } } + + protected String loadExpectedPlan(String fileName) throws IOException { + String prefix; + if (isCalciteEnabled()) { + if (isPushdownDisabled()) { + prefix = "expectedOutput/calcite_no_pushdown/"; + } else { + prefix = "expectedOutput/calcite/"; + } + } else { + prefix = "expectedOutput/ppl/"; + } + return loadFromFile(prefix + fileName); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java index be2cae9c843..b760a9c5546 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java @@ -195,9 +195,9 @@ public void testSortWithDescMultipleFields() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | sort 4 age, - account_number desc | fields age, account_number", + "source=%s | sort 4 age desc, account_number desc | fields age, account_number", TEST_INDEX_BANK)); - verifyOrder(result, rows(39, 25), rows(36, 6), rows(36, 20), rows(34, 32)); + verifyOrder(result, rows(39, 25), rows(36, 20), rows(36, 6), rows(34, 32)); } @Test @@ -241,7 +241,7 @@ public void testSortWithAscMultipleFields() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | sort age, account_number asc | fields age, account_number", + "source=%s | sort age asc, account_number asc | fields age, account_number", TEST_INDEX_BANK)); verifyOrder( result, @@ -253,4 +253,72 @@ public void testSortWithAscMultipleFields() throws IOException { rows(36, 20), rows(39, 25)); } + + @Test + public void testSortMixingPrefixWithDefault() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | sort +age, account_number, -balance | fields age, account_number," + + " balance", + TEST_INDEX_BANK)); + verifyOrder( + result, + rows(28, 13, 32838), + rows(32, 1, 39225), + rows(33, 18, 4180), + rows(34, 32, 48086), + rows(36, 6, 5686), + rows(36, 20, 16418), + rows(39, 25, 40540)); + } + + @Test + public void testSortMixingSuffixWithDefault() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | sort age, account_number desc, balance | fields age," + + " account_number, balance", + TEST_INDEX_BANK)); + verifyOrder( + result, + rows(28, 13, 32838), + rows(32, 1, 39225), + rows(33, 18, 4180), + rows(34, 32, 48086), + rows(36, 20, 16418), + rows(36, 6, 5686), + rows(39, 25, 40540)); + } + + @Test + public void testSortAllDefaultFields() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | sort age, account_number | fields age, account_number", + TEST_INDEX_BANK)); + verifyOrder( + result, + rows(28, 13), + rows(32, 1), + rows(33, 18), + rows(34, 32), + rows(36, 6), + rows(36, 20), + rows(39, 25)); + } + + @Test + public void testHeadThenSort() throws IOException { + JSONObject result = + executeQuery(String.format("source=%s | head 2 | sort age | fields age", TEST_INDEX_BANK)); + if (isPushdownDisabled()) { + // Pushdown is disabled, it will retrieve the first 2 docs since there's only 1 shard. + verifyOrder(result, rows(32), rows(36)); + } else { + verifyOrder(result, rows(28), rows(32)); + } + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java index afa5c787d7c..062880411ef 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java @@ -166,6 +166,7 @@ private Settings defaultSettings() { private final Map defaultSettings = new ImmutableMap.Builder() .put(Key.QUERY_SIZE_LIMIT, 200) + .put(Key.QUERY_BUCKET_SIZE, 1000) .put(Key.SQL_CURSOR_KEEP_ALIVE, TimeValue.timeValueMinutes(1)) .put(Key.FIELD_TYPE_TOLERANCE, true) .build(); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java index d04d2753673..c2b3ec0b407 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java @@ -608,6 +608,26 @@ public void testStatsPercentile() throws IOException { verifyDataRows(response, rows(32838)); } + @Test + public void testStatsPercentileWithMin() throws IOException { + JSONObject response = + executeQuery( + String.format( + "source=%s | eval decimal=ceil(balance/100000.0) | stats percentile(decimal, 50)," + + " min(decimal)", + TEST_INDEX_BANK)); + String returnType = "bigint"; + if (isCalciteEnabled()) { + returnType = "double"; + } + + verifySchema( + response, + schema("percentile(decimal, 50)", null, returnType), + schema("min(decimal)", null, returnType)); + verifyDataRows(response, rows(1, 1)); + } + @Test public void testStatsPercentileWithNull() throws IOException { JSONObject response = @@ -764,4 +784,26 @@ public void testStatsByCounts() throws IOException { rows(493, 493, 493, 493, 493, 493, 493, "F"), rows(507, 507, 507, 507, 507, 507, 507, "M")); } + + @Test + public void testStatsByDependentGroupFields() throws IOException { + JSONObject response = + executeQuery( + String.format( + "source=%s" + + "| eval age1 = age * 10, age2 = age + 10, age3 = 10" + + "| stats count() as cnt by age1, age2, age3, age" + + "| sort - cnt" + + "| head 3", + TEST_INDEX_ACCOUNT)); + verifySchema( + response, + schema("cnt", null, "bigint"), + schema("age1", null, "bigint"), + schema("age2", null, "bigint"), + schema("age3", null, "int"), + schema("age", null, "bigint")); + verifyDataRows( + response, rows(61, 310, 41, 10, 31), rows(60, 390, 49, 10, 39), rows(59, 260, 36, 10, 26)); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/SQLConcurrencyIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/SQLConcurrencyIT.java new file mode 100644 index 00000000000..76745d65695 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/sql/SQLConcurrencyIT.java @@ -0,0 +1,110 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.sql; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.legacy.TestsConstants; + +/** Verifies SQL aggregations remain stable under concurrent execution. */ +public class SQLConcurrencyIT extends SQLIntegTestCase { + + private static final int THREAD_POOL_SIZE = 256; + private static final long QUERY_TIMEOUT_SECONDS = 60; + + @Override + public void init() throws IOException { + loadIndex(Index.ACCOUNT); + } + + @Test + public void aggregationsHandleConcurrentSqlQueries() throws Exception { + String countQuery = + String.format("SELECT COUNT(age) AS cnt FROM %s", TestsConstants.TEST_INDEX_ACCOUNT); + String sumQuery = + String.format( + "SELECT SUM(balance) AS total_sales FROM %s", TestsConstants.TEST_INDEX_ACCOUNT); + + long expectedCount = extractNumber(executeJdbcRequest(countQuery)).longValue(); + double expectedSum = extractNumber(executeJdbcRequest(sumQuery)).doubleValue(); + + runConcurrentRound(countQuery, expectedCount, sumQuery, expectedSum); + } + + private void runConcurrentRound( + String countQuery, long expectedCount, String sumQuery, double expectedSum) throws Exception { + ExecutorService executor = Executors.newFixedThreadPool(THREAD_POOL_SIZE); + List> tasks = new ArrayList<>(); + + for (int i = 0; i < THREAD_POOL_SIZE; i++) { + tasks.add(countTask(countQuery, expectedCount)); + tasks.add(sumTask(sumQuery, expectedSum)); + } + + Collections.shuffle(tasks, ThreadLocalRandom.current()); + + List> futures = new ArrayList<>(); + try { + for (Callable task : tasks) { + futures.add(executor.submit(task)); + } + waitForTasks(futures); + } finally { + executor.shutdown(); + if (!executor.awaitTermination(QUERY_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + executor.shutdownNow(); + } + } + } + + private Callable countTask(String query, long expected) { + return () -> { + long actual = extractNumber(executeJdbcRequest(query)).longValue(); + assertEquals("Unexpected COUNT result", expected, actual); + return null; + }; + } + + private Callable sumTask(String query, double expected) { + return () -> { + double actual = extractNumber(executeJdbcRequest(query)).doubleValue(); + assertEquals("Unexpected SUM result", expected, actual, 1e-6); + return null; + }; + } + + private void waitForTasks(List> tasks) throws Exception { + for (Future task : tasks) { + task.get(QUERY_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + tasks.clear(); + } + + private Number extractNumber(JSONObject response) { + JSONArray rows = response.getJSONArray("datarows"); + assertEquals("Expected a single row", 1, rows.length()); + JSONArray row = rows.getJSONArray(0); + assertEquals("Expected a single column", 1, row.length()); + Object value = row.get(0); + assertTrue("Expected numeric result but got " + value, value instanceof Number); + return (Number) value; + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java index 83708840bff..01daded897d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java @@ -167,6 +167,7 @@ private Settings defaultSettings() { private final Map defaultSettings = new ImmutableMap.Builder() .put(Key.QUERY_SIZE_LIMIT, 200) + .put(Key.QUERY_BUCKET_SIZE, 1000) .put(Key.SQL_CURSOR_KEEP_ALIVE, TimeValue.timeValueMinutes(1)) .put(Key.FIELD_TYPE_TOLERANCE, true) .build(); diff --git a/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java b/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java index 929397594f0..b1ec25b485e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java @@ -405,6 +405,12 @@ public static void assertJsonEquals(String expected, String actual) { JsonParser.parseString(eliminatePid(actual))); } + public static void assertJsonEquals(String message, String expected, String actual) { + assertEquals(message, + JsonParser.parseString(eliminatePid(expected)), + JsonParser.parseString(eliminatePid(actual))); + } + /** Compare two JSON string are equals with ignoring the RelNode id in the Calcite plan. */ public static void assertJsonEqualsIgnoreId(String expected, String actual) { assertJsonEquals(cleanUpId(expected), cleanUpId(actual)); @@ -426,8 +432,8 @@ private static String eliminatePid(String s) { return s.replaceAll("pitId=[^,]+,", "pitId=*,"); } - public static void assertYamlEqualsJsonIgnoreId(String expectedYaml, String actualJson) { - String cleanedYaml = cleanUpYaml(jsonToYaml(actualJson)); + public static void assertYamlEqualsIgnoreId(String expectedYaml, String actualYaml) { + String cleanedYaml = cleanUpYaml(actualYaml); assertYamlEquals(expectedYaml, cleanedYaml); } diff --git a/integ-test/src/test/java/org/opensearch/sql/util/RetryProcessor.java b/integ-test/src/test/java/org/opensearch/sql/util/RetryProcessor.java index e8f12aac566..3c2f3f060dc 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/RetryProcessor.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/RetryProcessor.java @@ -35,6 +35,7 @@ public void evaluate() throws Throwable { } catch (Throwable t) { lastException = t; LOG.info("Retrying {} {} times", description.getDisplayName(), (i + 1)); + Thread.sleep(3000); } } assert lastException != null; diff --git a/integ-test/src/test/resources/big5/queries/asc_sort_timestamp.ppl b/integ-test/src/test/resources/big5/queries/asc_sort_timestamp.ppl index 7582d40d69a..f66d590a44d 100644 --- a/integ-test/src/test/resources/big5/queries/asc_sort_timestamp.ppl +++ b/integ-test/src/test/resources/big5/queries/asc_sort_timestamp.ppl @@ -1,3 +1,18 @@ +/* +{ + "name": "asc_sort_timestamp", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "match_all": {} + }, + "sort" : [ + {"@timestamp" : "asc"} + ] + } +} + */ source = big5 | sort + `@timestamp` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/asc_sort_timestamp_can_match_shortcut.ppl b/integ-test/src/test/resources/big5/queries/asc_sort_timestamp_can_match_shortcut.ppl index aab85fb7c1b..de766526c63 100644 --- a/integ-test/src/test/resources/big5/queries/asc_sort_timestamp_can_match_shortcut.ppl +++ b/integ-test/src/test/resources/big5/queries/asc_sort_timestamp_can_match_shortcut.ppl @@ -1,3 +1,21 @@ +/* +{ + "name": "asc_sort_timestamp_can_match_shortcut", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "track_total_hits": false, + "query": { + "match": { + "process.name": "kernel" + } + }, + "sort" : [ + {"@timestamp" : "asc"} + ] + } +} +*/ source = big5 process.name=kernel | sort + `@timestamp` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/asc_sort_timestamp_no_can_match_shortcut.ppl b/integ-test/src/test/resources/big5/queries/asc_sort_timestamp_no_can_match_shortcut.ppl index aab85fb7c1b..4957fcd485f 100644 --- a/integ-test/src/test/resources/big5/queries/asc_sort_timestamp_no_can_match_shortcut.ppl +++ b/integ-test/src/test/resources/big5/queries/asc_sort_timestamp_no_can_match_shortcut.ppl @@ -1,3 +1,24 @@ +/* +{ + "name": "asc_sort_timestamp_no_can_match_shortcut", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "request-params" : { + "pre_filter_shard_size" : 100000 + }, + "body": { + "track_total_hits": false, + "query": { + "match": { + "process.name": "kernel" + } + }, + "sort" : [ + {"@timestamp" : "asc"} + ] + } +} +*/ source = big5 process.name=kernel | sort + `@timestamp` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/asc_sort_with_after_timestamp.ppl b/integ-test/src/test/resources/big5/queries/asc_sort_with_after_timestamp.ppl index 7582d40d69a..02e96b4f731 100644 --- a/integ-test/src/test/resources/big5/queries/asc_sort_with_after_timestamp.ppl +++ b/integ-test/src/test/resources/big5/queries/asc_sort_with_after_timestamp.ppl @@ -1,3 +1,24 @@ +/* +{ + "name": "asc_sort_with_after_timestamp", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "track_total_hits": false, + "query": { + "match_all": {} + }, + "sort" : [ + {"@timestamp" : "asc"} + ], +{% if distribution_version.split('.') | map('int') | list < "6.0.0".split('.') | map('int') | list or distribution_version.split('.') | map('int') | list >= "7.0.0".split('.') | map('int') | list %} + "search_after": ["2023-01-01T23:59:58.000Z"] +{% else %} + "search_after": [1673049598] +{% endif %} + } +} +*/ source = big5 | sort + `@timestamp` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/cardinality_agg_high.ppl b/integ-test/src/test/resources/big5/queries/cardinality_agg_high.ppl new file mode 100644 index 00000000000..0e8c3dec630 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/cardinality_agg_high.ppl @@ -0,0 +1,22 @@ +/* +{ + "name": "cardinality-agg-high", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "agent": { + "cardinality": { + "field": "agent.name" + {% if distribution_version.split('.') | map('int') | list >= "2.19.1".split('.') | map('int') | list and distribution_version.split('.') | map('int') | list < "6.0.0".split('.') | map('int') | list %} + , "execution_hint": "ordinals" + {% endif %} + } + } + } + } +} +*/ +source = big5 +| stats dc(`agent.name`) \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/cardinality_agg_high_2.ppl b/integ-test/src/test/resources/big5/queries/cardinality_agg_high_2.ppl new file mode 100644 index 00000000000..93130873f90 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/cardinality_agg_high_2.ppl @@ -0,0 +1,21 @@ +/* +{ + "name": "cardinality-agg-high-2", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "request-timeout": 1800, + "body": { + "size": 0, + "aggs": { + "agent": { + "cardinality": { + "field": "event.id", + "execution_hint":"ordinals" + } + } + } + } +} +*/ +source = big5 +| stats dc(`event.id`) \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/cardinality_agg_low.ppl b/integ-test/src/test/resources/big5/queries/cardinality_agg_low.ppl new file mode 100644 index 00000000000..ca6c8b214c3 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/cardinality_agg_low.ppl @@ -0,0 +1,19 @@ +/* +{ + "name": "cardinality-agg-low", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "region": { + "cardinality": { + "field": "cloud.region" + } + } + } + } +} +*/ +source = big5 +| stats dc(`cloud.region`) \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/composite_date_histogram_daily.ppl b/integ-test/src/test/resources/big5/queries/composite_date_histogram_daily.ppl index caa27c86fba..656289b0603 100644 --- a/integ-test/src/test/resources/big5/queries/composite_date_histogram_daily.ppl +++ b/integ-test/src/test/resources/big5/queries/composite_date_histogram_daily.ppl @@ -1,3 +1,34 @@ +/* +{ + "name": "composite-date_histogram-daily", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "query": { + "range": { + "@timestamp": { + "gte": "2022-12-30T00:00:00", + "lt": "2023-01-07T12:00:00" + } + } + }, + "aggs": { + "logs": { + "composite": { + "sources": [ + {% if distribution_version.split('.') | map('int') | list < "6.0.0".split('.') | map('int') | list or distribution_version.split('.') | map('int') | list >= "7.0.0".split('.') | map('int') | list %} + { "date": { "date_histogram": { "field": "@timestamp", "calendar_interval": "day" } } } + {% else %} + { "date": { "date_histogram": { "field": "@timestamp", "interval": "day" } } } + {% endif %} + ] + } + } + } + } +} +*/ source = big5 | where `@timestamp` >= '2022-12-30 00:00:00' and `@timestamp` < '2023-01-07 12:00:00' | stats count() by span(`@timestamp`, 1d) \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/composite_terms.ppl b/integ-test/src/test/resources/big5/queries/composite_terms.ppl index 859e3c87e54..07edca09e69 100644 --- a/integ-test/src/test/resources/big5/queries/composite_terms.ppl +++ b/integ-test/src/test/resources/big5/queries/composite_terms.ppl @@ -1,4 +1,32 @@ +/* +{ + "name": "composite-terms", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-02T00:00:00", + "lt": "2023-01-02T10:00:00" + } + } + }, + "aggs": { + "logs": { + "composite": { + "sources": [ + { "process_name": { "terms": { "field": "process.name", "order": "desc" }}}, + { "cloud_region": { "terms": { "field": "cloud.region", "order": "asc" }}} + ] + } + } + } + } +} +*/ source = big5 -| where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' +| where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-02 10:00:00' | stats count() by `process.name`, `cloud.region` | sort - `process.name`, + `cloud.region` \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/composite_terms_keyword.ppl b/integ-test/src/test/resources/big5/queries/composite_terms_keyword.ppl index 5eb03e5fe2a..42b8c9585a4 100644 --- a/integ-test/src/test/resources/big5/queries/composite_terms_keyword.ppl +++ b/integ-test/src/test/resources/big5/queries/composite_terms_keyword.ppl @@ -1,4 +1,33 @@ +/* +{ + "name": "composite_terms-keyword", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-02T00:00:00", + "lt": "2023-01-02T10:00:00" + } + } + }, + "aggs": { + "logs": { + "composite": { + "sources": [ + { "process_name": { "terms": { "field": "process.name", "order": "desc" }}}, + { "cloud_region": { "terms": { "field": "cloud.region", "order": "asc" }}}, + { "cloudstream": { "terms": { "field": "aws.cloudwatch.log_stream", "order": "asc" }}} + ] + } + } + } + } +} +*/ source = big5 -| where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' +| where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-02 10:00:00' | stats count() by `process.name`, `cloud.region`, `aws.cloudwatch.log_stream` | sort - `process.name`, + `cloud.region`, + `aws.cloudwatch.log_stream` \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/date_histogram_hourly_agg.ppl b/integ-test/src/test/resources/big5/queries/date_histogram_hourly_agg.ppl index 054b915b335..4a340cf04d2 100644 --- a/integ-test/src/test/resources/big5/queries/date_histogram_hourly_agg.ppl +++ b/integ-test/src/test/resources/big5/queries/date_histogram_hourly_agg.ppl @@ -1,2 +1,24 @@ +/* +{ + "name": "date_histogram_hourly_agg", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "by_hour": { + "date_histogram": { + "field": "@timestamp", + {% if distribution_version.split('.') | map('int') | list < "6.0.0".split('.') | map('int') | list or distribution_version.split('.') | map('int') | list >= "7.0.0".split('.') | map('int') | list %} + "calendar_interval": "hour" + {% else %} + "interval": "hour" + {% endif %} + } + } + } + } +} +*/ source = big5 | stats count() by span(`@timestamp`, 1h) \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/date_histogram_minute_agg.ppl b/integ-test/src/test/resources/big5/queries/date_histogram_minute_agg.ppl index b9fd72abfb5..e7c647f213e 100644 --- a/integ-test/src/test/resources/big5/queries/date_histogram_minute_agg.ppl +++ b/integ-test/src/test/resources/big5/queries/date_histogram_minute_agg.ppl @@ -1,3 +1,33 @@ +/* +{ + "name": "date_histogram_minute_agg", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-01T00:00:00", + "lt": "2023-01-03T00:00:00" + } + } + }, + "aggs": { + "by_hour": { + "date_histogram": { + "field": "@timestamp", + {% if distribution_version.split('.') | map('int') | list < "6.0.0".split('.') | map('int') | list or distribution_version.split('.') | map('int') | list >= "7.0.0".split('.') | map('int') | list %} + "calendar_interval": "minute" + {% else %} + "interval": "minute" + {% endif %} + } + } + } + } +} +*/ source = big5 | where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' | stats count() by span(`@timestamp`, 1m) \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/default.ppl b/integ-test/src/test/resources/big5/queries/default.ppl index 6b63c414ac0..2aed0d33141 100644 --- a/integ-test/src/test/resources/big5/queries/default.ppl +++ b/integ-test/src/test/resources/big5/queries/default.ppl @@ -1,2 +1,14 @@ +/* +{ + "name": "match-all", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "match_all": {} + } + } +} +*/ source = big5 | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/desc_sort_timestamp.ppl b/integ-test/src/test/resources/big5/queries/desc_sort_timestamp.ppl index af3445efdff..91ff8c5ad01 100644 --- a/integ-test/src/test/resources/big5/queries/desc_sort_timestamp.ppl +++ b/integ-test/src/test/resources/big5/queries/desc_sort_timestamp.ppl @@ -1,3 +1,18 @@ +/* +{ + "name": "desc_sort_timestamp", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "match_all": {} + }, + "sort" : [ + {"@timestamp" : "desc"} + ] + } +} +*/ source = big5 | sort - `@timestamp` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/desc_sort_timestamp_can_match_shortcut.ppl b/integ-test/src/test/resources/big5/queries/desc_sort_timestamp_can_match_shortcut.ppl index 84205ef61fc..844784cbde3 100644 --- a/integ-test/src/test/resources/big5/queries/desc_sort_timestamp_can_match_shortcut.ppl +++ b/integ-test/src/test/resources/big5/queries/desc_sort_timestamp_can_match_shortcut.ppl @@ -1,3 +1,21 @@ +/* +{ + "name": "desc_sort_timestamp_can_match_shortcut", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "track_total_hits": false, + "query": { + "match": { + "process.name": "kernel" + } + }, + "sort" : [ + {"@timestamp" : "desc"} + ] + } +} +*/ source = big5 process.name=kernel | sort - `@timestamp` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/desc_sort_timestamp_no_can_match_shortcut.ppl b/integ-test/src/test/resources/big5/queries/desc_sort_timestamp_no_can_match_shortcut.ppl index 84205ef61fc..b8516925f18 100644 --- a/integ-test/src/test/resources/big5/queries/desc_sort_timestamp_no_can_match_shortcut.ppl +++ b/integ-test/src/test/resources/big5/queries/desc_sort_timestamp_no_can_match_shortcut.ppl @@ -1,3 +1,24 @@ +/* +{ + "name": "desc_sort_timestamp_no_can_match_shortcut", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "request-params" : { + "pre_filter_shard_size" : 100000 + }, + "body": { + "track_total_hits": false, + "query": { + "match": { + "process.name": "kernel" + } + }, + "sort" : [ + {"@timestamp" : "desc"} + ] + } +} +*/ source = big5 process.name=kernel | sort - `@timestamp` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/desc_sort_with_after_timestamp.ppl b/integ-test/src/test/resources/big5/queries/desc_sort_with_after_timestamp.ppl index af3445efdff..869514e9f8f 100644 --- a/integ-test/src/test/resources/big5/queries/desc_sort_with_after_timestamp.ppl +++ b/integ-test/src/test/resources/big5/queries/desc_sort_with_after_timestamp.ppl @@ -1,3 +1,24 @@ +/* +{ + "name": "desc_sort_with_after_timestamp", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "track_total_hits": false, + "query": { + "match_all": {} + }, + "sort" : [ + {"@timestamp" : "desc"} + ], +{% if distribution_version.split('.') | map('int') | list < "6.0.0".split('.') | map('int') | list or distribution_version.split('.') | map('int') | list >= "7.0.0".split('.') | map('int') | list %} + "search_after": ["2023-01-01T23:59:58.000Z"] +{% else %} + "search_after": [1673049598] +{% endif %} + } +} +*/ source = big5 | sort - `@timestamp` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/keyword_in_range.ppl b/integ-test/src/test/resources/big5/queries/keyword_in_range.ppl index 1c717e8472c..331a2212d9e 100644 --- a/integ-test/src/test/resources/big5/queries/keyword_in_range.ppl +++ b/integ-test/src/test/resources/big5/queries/keyword_in_range.ppl @@ -1,4 +1,31 @@ +/* +{ + "name": "keyword-in-range", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "bool": { + "must": [ + { + "range": { + "@timestamp": { + "gte": "2023-01-01T00:00:00", + "lt": "2023-01-03T00:00:00" + } + } + }, + { + "match": { + "process.name": "kernel" + } + } + ] + } + } + } +} +*/ source = big5 process.name=kernel -| where `@timestamp` >= '2023-01-01 00:00:00' - and `@timestamp` < '2023-01-03 00:00:00' +| where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/keyword_terms.ppl b/integ-test/src/test/resources/big5/queries/keyword_terms.ppl index 99353b5299f..1329aaf6570 100644 --- a/integ-test/src/test/resources/big5/queries/keyword_terms.ppl +++ b/integ-test/src/test/resources/big5/queries/keyword_terms.ppl @@ -1,3 +1,21 @@ +/* +{ + "name": "keyword-terms", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "station": { + "terms": { + "field": "aws.cloudwatch.log_stream", + "size": 500 + } + } + } + } +} +*/ source = big5 | stats count() as station by `aws.cloudwatch.log_stream` | sort - station diff --git a/integ-test/src/test/resources/big5/queries/keyword_terms_low_cardinality.ppl b/integ-test/src/test/resources/big5/queries/keyword_terms_low_cardinality.ppl index 02e335723d3..11adb833804 100644 --- a/integ-test/src/test/resources/big5/queries/keyword_terms_low_cardinality.ppl +++ b/integ-test/src/test/resources/big5/queries/keyword_terms_low_cardinality.ppl @@ -1,4 +1,22 @@ +/* +{ + "name": "keyword-terms-low-cardinality", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "country": { + "terms": { + "field": "aws.cloudwatch.log_stream", + "size": 50 + } + } + } + } +} +*/ source = big5 | stats count() as country by `aws.cloudwatch.log_stream` | sort - country -| head 100 \ No newline at end of file +| head 50 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/multi_terms_keyword.ppl b/integ-test/src/test/resources/big5/queries/multi_terms_keyword.ppl index a148a8bbc90..d88f2cf7ce3 100644 --- a/integ-test/src/test/resources/big5/queries/multi_terms_keyword.ppl +++ b/integ-test/src/test/resources/big5/queries/multi_terms_keyword.ppl @@ -1,4 +1,38 @@ +/* +{ + "name": "multi_terms-keyword", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "request-timeout": 7200, + "body":{ + "size": 0, + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-05T00:00:00", + "lt": "2023-01-05T05:00:00" + } + } + }, + "aggs": { + "important_terms": { + "multi_terms": { + "terms": [ + { + "field": "process.name" + }, + { + "field": "cloud.region" + } + ] + } + } + } + } +} +*/ source = big5 -| where `@timestamp` >= '2022-12-30 00:00:00' and `@timestamp` < '2023-01-01 03:00:00' -| stats count() by `process.name`, `event.id`, `cloud.region` -| sort - `count()` \ No newline at end of file +| where `@timestamp` >= '2023-01-05 00:00:00' and `@timestamp` < '2023-01-05 05:00:00' +| stats count() by `process.name`, `cloud.region` +| sort - `count()` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/optimized/cardinality_agg_high.ppl b/integ-test/src/test/resources/big5/queries/optimized/cardinality_agg_high.ppl new file mode 100644 index 00000000000..7d291cd766e --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/optimized/cardinality_agg_high.ppl @@ -0,0 +1,22 @@ +/* +{ + "name": "cardinality-agg-high", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "agent": { + "cardinality": { + "field": "agent.name" + {% if distribution_version.split('.') | map('int') | list >= "2.19.1".split('.') | map('int') | list and distribution_version.split('.') | map('int') | list < "6.0.0".split('.') | map('int') | list %} + , "execution_hint": "ordinals" + {% endif %} + } + } + } + } +} +*/ +source = big5 +| stats bucket_nullable = false dc(`agent.name`) \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/optimized/cardinality_agg_high_2.ppl b/integ-test/src/test/resources/big5/queries/optimized/cardinality_agg_high_2.ppl new file mode 100644 index 00000000000..7ff01915a8c --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/optimized/cardinality_agg_high_2.ppl @@ -0,0 +1,21 @@ +/* +{ + "name": "cardinality-agg-high-2", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "request-timeout": 1800, + "body": { + "size": 0, + "aggs": { + "agent": { + "cardinality": { + "field": "event.id", + "execution_hint":"ordinals" + } + } + } + } +} +*/ +source = big5 +| stats bucket_nullable = false dc(`event.id`) \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/optimized/cardinality_agg_low.ppl b/integ-test/src/test/resources/big5/queries/optimized/cardinality_agg_low.ppl new file mode 100644 index 00000000000..f763f61a77d --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/optimized/cardinality_agg_low.ppl @@ -0,0 +1,19 @@ +/* +{ + "name": "cardinality-agg-low", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "region": { + "cardinality": { + "field": "cloud.region" + } + } + } + } +} +*/ +source = big5 +| stats bucket_nullable = false dc(`cloud.region`) \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/optimized/composite_terms.ppl b/integ-test/src/test/resources/big5/queries/optimized/composite_terms.ppl new file mode 100644 index 00000000000..97897e227de --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/optimized/composite_terms.ppl @@ -0,0 +1,32 @@ +/* +{ + "name": "composite-terms", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-02T00:00:00", + "lt": "2023-01-02T10:00:00" + } + } + }, + "aggs": { + "logs": { + "composite": { + "sources": [ + { "process_name": { "terms": { "field": "process.name", "order": "desc" }}}, + { "cloud_region": { "terms": { "field": "cloud.region", "order": "asc" }}} + ] + } + } + } + } +} +*/ +source = big5 +| where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-02 10:00:00' +| stats bucket_nullable = false count() by `process.name`, `cloud.region` +| sort - `process.name`, + `cloud.region` \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/optimized/composite_terms_keyword.ppl b/integ-test/src/test/resources/big5/queries/optimized/composite_terms_keyword.ppl new file mode 100644 index 00000000000..04d12b4fb0e --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/optimized/composite_terms_keyword.ppl @@ -0,0 +1,33 @@ +/* +{ + "name": "composite_terms-keyword", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-02T00:00:00", + "lt": "2023-01-02T10:00:00" + } + } + }, + "aggs": { + "logs": { + "composite": { + "sources": [ + { "process_name": { "terms": { "field": "process.name", "order": "desc" }}}, + { "cloud_region": { "terms": { "field": "cloud.region", "order": "asc" }}}, + { "cloudstream": { "terms": { "field": "aws.cloudwatch.log_stream", "order": "asc" }}} + ] + } + } + } + } +} +*/ +source = big5 +| where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-02 10:00:00' +| stats bucket_nullable = false count() by `process.name`, `cloud.region`, `aws.cloudwatch.log_stream` +| sort - `process.name`, + `cloud.region`, + `aws.cloudwatch.log_stream` \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/optimized/keyword_terms.ppl b/integ-test/src/test/resources/big5/queries/optimized/keyword_terms.ppl new file mode 100644 index 00000000000..062eea752bb --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/optimized/keyword_terms.ppl @@ -0,0 +1,22 @@ +/* +{ + "name": "keyword-terms", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "station": { + "terms": { + "field": "aws.cloudwatch.log_stream", + "size": 500 + } + } + } + } +} +*/ +source = big5 +| stats bucket_nullable = false count() as station by `aws.cloudwatch.log_stream` +| sort - station +| head 500 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/optimized/keyword_terms_low_cardinality.ppl b/integ-test/src/test/resources/big5/queries/optimized/keyword_terms_low_cardinality.ppl new file mode 100644 index 00000000000..71812820cab --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/optimized/keyword_terms_low_cardinality.ppl @@ -0,0 +1,22 @@ +/* +{ + "name": "keyword-terms-low-cardinality", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "country": { + "terms": { + "field": "aws.cloudwatch.log_stream", + "size": 50 + } + } + } + } +} +*/ +source = big5 +| stats bucket_nullable = false count() as country by `aws.cloudwatch.log_stream` +| sort - country +| head 50 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/optimized/multi_terms_keyword.ppl b/integ-test/src/test/resources/big5/queries/optimized/multi_terms_keyword.ppl new file mode 100644 index 00000000000..9221e728e09 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/optimized/multi_terms_keyword.ppl @@ -0,0 +1,38 @@ +/* +{ + "name": "multi_terms-keyword", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "request-timeout": 7200, + "body":{ + "size": 0, + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-05T00:00:00", + "lt": "2023-01-05T05:00:00" + } + } + }, + "aggs": { + "important_terms": { + "multi_terms": { + "terms": [ + { + "field": "process.name" + }, + { + "field": "cloud.region" + } + ] + } + } + } + } +} +*/ +source = big5 +| where `@timestamp` >= '2023-01-05 00:00:00' and `@timestamp` < '2023-01-05 05:00:00' +| stats bucket_nullable = false count() by `process.name`, `cloud.region` +| sort - `count()` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/optimized/range_auto_date_histo.ppl b/integ-test/src/test/resources/big5/queries/optimized/range_auto_date_histo.ppl new file mode 100644 index 00000000000..6711ce74d58 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/optimized/range_auto_date_histo.ppl @@ -0,0 +1,59 @@ +/* +{ + "name": "range-auto-date-histo", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "tmax": { + "range": { + "field": "metrics.size", + "ranges": [ + { + "to": -10 + }, + { + "from": -10, + "to": 10 + }, + { + "from": 10, + "to": 100 + }, + { + "from": 100, + "to": 1000 + }, + { + "from": 1000, + "to": 2000 + }, + { + "from": 2000 + } + ] + }, + "aggs": { + "date": { + "auto_date_histogram": { + "field": "@timestamp", + "buckets": 20 + } + } + } + } + } + } +} +*/ +source = big5 +| eval range_bucket = case( + `metrics.size` < -10, 'range_1', + `metrics.size` >= -10 and `metrics.size` < 10, 'range_2', + `metrics.size` >= 10 and `metrics.size` < 100, 'range_3', + `metrics.size` >= 100 and `metrics.size` < 1000, 'range_4', + `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_5', + `metrics.size` >= 2000, 'range_6') +| bin @timestamp bins=20 +| stats count() by range_bucket, @timestamp \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/optimized/range_auto_date_histo_with_metrics.ppl b/integ-test/src/test/resources/big5/queries/optimized/range_auto_date_histo_with_metrics.ppl new file mode 100644 index 00000000000..09e83a63ebe --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/optimized/range_auto_date_histo_with_metrics.ppl @@ -0,0 +1,67 @@ +/* +{ + "name": "range-auto-date-histo-with-metrics", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "request-timeout": 7200, + "body": { + "size": 0, + "aggs": { + "tmax": { + "range": { + "field": "metrics.size", + "ranges": [ + { + "to": 100 + }, + { + "from": 100, + "to": 1000 + }, + { + "from": 1000, + "to": 2000 + }, + { + "from": 2000 + } + ] + }, + "aggs": { + "date": { + "auto_date_histogram": { + "field": "@timestamp", + "buckets": 10 + }, + "aggs": { + "tmin": { + "min": { + "field": "metrics.tmin" + } + }, + "tavg": { + "avg": { + "field": "metrics.size" + } + }, + "tmax": { + "max": { + "field": "metrics.size" + } + } + } + } + } + } + } + } +} +*/ +source = big5 +| eval range_bucket = case( + `metrics.size` < 100, 'range_1', + `metrics.size` >= 100 and `metrics.size` < 1000, 'range_2', + `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_3', + `metrics.size` >= 2000, 'range_4') +| bin @timestamp bins=10 +| stats min(`metrics.tmin`) as tmin, avg(`metrics.size`) as tavg, max(`metrics.size`) as tmax by range_bucket, @timestamp \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/query_string_on_message.ppl b/integ-test/src/test/resources/big5/queries/query_string_on_message.ppl index 2f0d33a6a3b..6a11bd84867 100644 --- a/integ-test/src/test/resources/big5/queries/query_string_on_message.ppl +++ b/integ-test/src/test/resources/big5/queries/query_string_on_message.ppl @@ -1,2 +1,16 @@ -source = big5 | where query_string(['message'], 'shield AND carp AND shark') +/* +{ + "name": "query-string-on-message", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "query_string": { + "query": "message: monkey jackal bear" + } + } + } +} +*/ +source = big5 message=monkey OR message=jackal OR message=bear | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/query_string_on_message_filtered.ppl b/integ-test/src/test/resources/big5/queries/query_string_on_message_filtered.ppl index 3abd5cea089..cb8be286a45 100644 --- a/integ-test/src/test/resources/big5/queries/query_string_on_message_filtered.ppl +++ b/integ-test/src/test/resources/big5/queries/query_string_on_message_filtered.ppl @@ -1,4 +1,32 @@ -source = big5 message=shield message=carp message=shark -| where `@timestamp` >= '2023-01-01 00:00:00' - and `@timestamp` < '2023-01-03 00:00:00' +/* +{ + "name": "query-string-on-message-filtered", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "bool": { + "must": [ + { + "range": { + "@timestamp": { + "gte": "2023-01-03T00:00:00", + "lt": "2023-01-03T10:00:00" + } + } + }, + { + "query_string": { + "query": "message: monkey jackal bear" + } + } + ] + } + } + } +} +*/ +source = big5 +| where `@timestamp` >= '2023-01-03 00:00:00' and `@timestamp` < '2023-01-03 10:00:00' + AND query_string(['message'], 'monkey jackal bear') | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/query_string_on_message_filtered_sorted_num.ppl b/integ-test/src/test/resources/big5/queries/query_string_on_message_filtered_sorted_num.ppl index 8baf49c987c..732c57c83d6 100644 --- a/integ-test/src/test/resources/big5/queries/query_string_on_message_filtered_sorted_num.ppl +++ b/integ-test/src/test/resources/big5/queries/query_string_on_message_filtered_sorted_num.ppl @@ -1,5 +1,40 @@ -source = big5 | where query_string(['message'], 'shield AND carp AND shark') -| where `@timestamp` >= '2023-01-01 00:00:00' - and `@timestamp` < '2023-01-03 00:00:00' -| sort - `metrics.size` +/* +{ + "name": "query-string-on-message-filtered-sorted-num", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "bool": { + "must": [ + { + "range": { + "@timestamp": { + "gte": "2023-01-03T00:00:00", + "lt": "2023-01-03T10:00:00" + } + } + }, + { + "query_string": { + "query": "message: monkey jackal bear" + } + } + ] + } + }, + "sort": [ + { + "@timestamp": { + "order": "asc" + } + } + ] + } +} +*/ +source = big5 +| where `@timestamp` >= '2023-01-03 00:00:00' and `@timestamp` < '2023-01-03 10:00:00' + AND query_string(['message'], 'monkey jackal bear') +| sort `@timestamp` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range.ppl b/integ-test/src/test/resources/big5/queries/range.ppl index 74eae492541..b5480c6ba51 100644 --- a/integ-test/src/test/resources/big5/queries/range.ppl +++ b/integ-test/src/test/resources/big5/queries/range.ppl @@ -1,3 +1,20 @@ +/* +{ + "name": "range", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-01T00:00:00", + "lt": "2023-01-03T00:00:00" + } + } + } + } +} +*/ source = big5 | where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_agg_1.ppl b/integ-test/src/test/resources/big5/queries/range_agg_1.ppl new file mode 100644 index 00000000000..95280e697bb --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/range_agg_1.ppl @@ -0,0 +1,50 @@ +/* +{ + "name": "range-agg-1", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "tmax": { + "range": { + "field": "metrics.size", + "ranges": [ + { + "to": -10 + }, + { + "from": -10, + "to": 10 + }, + { + "from": 10, + "to": 100 + }, + { + "from": 100, + "to": 1000 + }, + { + "from": 1000, + "to": 2000 + }, + { + "from": 2000 + } + ] + } + } + } + } +} +*/ +source = big5 +| eval range_bucket = case( + `metrics.size` < -10, 'range_1', + `metrics.size` >= -10 and `metrics.size` < 10, 'range_2', + `metrics.size` >= 10 and `metrics.size` < 100, 'range_3', + `metrics.size` >= 100 and `metrics.size` < 1000, 'range_4', + `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_5', + `metrics.size` >= 2000, 'range_6') +| stats count() by range_bucket \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_agg_2.ppl b/integ-test/src/test/resources/big5/queries/range_agg_2.ppl new file mode 100644 index 00000000000..0988d688054 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/range_agg_2.ppl @@ -0,0 +1,40 @@ +/* +{ + "name": "range-agg-2", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "tmax": { + "range": { + "field": "metrics.size", + "ranges": [ + { + "to": 100 + }, + { + "from": 100, + "to": 1000 + }, + { + "from": 1000, + "to": 2000 + }, + { + "from": 2000 + } + ] + } + } + } + } +} +*/ +source = big5 +| eval range_bucket = case( + `metrics.size` < 100, 'range_1', + `metrics.size` >= 100 and `metrics.size` < 1000, 'range_2', + `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_3', + `metrics.size` >= 2000, 'range_4') +| stats count() by range_bucket \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_auto_date_histo.ppl b/integ-test/src/test/resources/big5/queries/range_auto_date_histo.ppl index 52a51cf7419..6126623a8b1 100644 --- a/integ-test/src/test/resources/big5/queries/range_auto_date_histo.ppl +++ b/integ-test/src/test/resources/big5/queries/range_auto_date_histo.ppl @@ -1,3 +1,52 @@ +/* +{ + "name": "range-auto-date-histo", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "tmax": { + "range": { + "field": "metrics.size", + "ranges": [ + { + "to": -10 + }, + { + "from": -10, + "to": 10 + }, + { + "from": 10, + "to": 100 + }, + { + "from": 100, + "to": 1000 + }, + { + "from": 1000, + "to": 2000 + }, + { + "from": 2000 + } + ] + }, + "aggs": { + "date": { + "auto_date_histogram": { + "field": "@timestamp", + "buckets": 20 + } + } + } + } + } + } +} +*/ source = big5 | eval range_bucket = case( `metrics.size` < -10, 'range_1', diff --git a/integ-test/src/test/resources/big5/queries/range_auto_date_histo_with_metrics.ppl b/integ-test/src/test/resources/big5/queries/range_auto_date_histo_with_metrics.ppl index 506978ace5b..d08c67823fa 100644 --- a/integ-test/src/test/resources/big5/queries/range_auto_date_histo_with_metrics.ppl +++ b/integ-test/src/test/resources/big5/queries/range_auto_date_histo_with_metrics.ppl @@ -1,10 +1,67 @@ +/* +{ + "name": "range-auto-date-histo-with-metrics", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "request-timeout": 7200, + "body": { + "size": 0, + "aggs": { + "tmax": { + "range": { + "field": "metrics.size", + "ranges": [ + { + "to": 100 + }, + { + "from": 100, + "to": 1000 + }, + { + "from": 1000, + "to": 2000 + }, + { + "from": 2000 + } + ] + }, + "aggs": { + "date": { + "auto_date_histogram": { + "field": "@timestamp", + "buckets": 10 + }, + "aggs": { + "tmin": { + "min": { + "field": "metrics.tmin" + } + }, + "tavg": { + "avg": { + "field": "metrics.size" + } + }, + "tmax": { + "max": { + "field": "metrics.size" + } + } + } + } + } + } + } + } +} +*/ source = big5 | eval range_bucket = case( - `metrics.size` < -10, 'range_1', - `metrics.size` >= -10 and `metrics.size` < 10, 'range_2', - `metrics.size` >= 10 and `metrics.size` < 100, 'range_3', - `metrics.size` >= 100 and `metrics.size` < 1000, 'range_4', - `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_5', - `metrics.size` >= 2000, 'range_6') + `metrics.size` < 100, 'range_1', + `metrics.size` >= 100 and `metrics.size` < 1000, 'range_2', + `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_3', + `metrics.size` >= 2000, 'range_4') | stats min(`metrics.tmin`) as tmin, avg(`metrics.size`) as tavg, max(`metrics.size`) as tmax by range_bucket, span(`@timestamp`, 1h) as auto_span | sort + range_bucket, + auto_span \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_field_conjunction_big_range_big_term_query.ppl b/integ-test/src/test/resources/big5/queries/range_field_conjunction_big_range_big_term_query.ppl index e6390a38cbf..905ad7f3dcc 100644 --- a/integ-test/src/test/resources/big5/queries/range_field_conjunction_big_range_big_term_query.ppl +++ b/integ-test/src/test/resources/big5/queries/range_field_conjunction_big_range_big_term_query.ppl @@ -1,5 +1,31 @@ +/* +{ + "name": "range_field_conjunction_big_range_big_term_query", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "bool": { + "must": [ + { + "term": { + "process.name": "systemd" + } + }, + { + "range": { + "metrics.size": { + "gte": 1, + "lte": 100 + } + } + } + ] + } + } + } +} +*/ source = big5 -| where `process.name` = 'systemd' - and `metrics.size` >= 1 - and `metrics.size` <= 1000 +| where `process.name` = 'systemd' and `metrics.size` >= 1 and `metrics.size` <= 100 | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_field_conjunction_small_range_big_term_query.ppl b/integ-test/src/test/resources/big5/queries/range_field_conjunction_small_range_big_term_query.ppl index f762da83896..1451f9a382f 100644 --- a/integ-test/src/test/resources/big5/queries/range_field_conjunction_small_range_big_term_query.ppl +++ b/integ-test/src/test/resources/big5/queries/range_field_conjunction_small_range_big_term_query.ppl @@ -1,3 +1,26 @@ +/* +{ + "name": "range_field_conjunction_small_range_big_term_query", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "bool": { + "must": [ + { + "range": { + "metrics.size": { + "gte": 20, + "lte": 30 + } + } + } + ] + } + } + } +} +*/ source = big5 -| where `metrics.size` >= 1 and `metrics.size` <= 42 +| where `metrics.size` >= 20 and `metrics.size` <= 30 | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_field_conjunction_small_range_small_term_query.ppl b/integ-test/src/test/resources/big5/queries/range_field_conjunction_small_range_small_term_query.ppl index 9d0742e122f..a6af46b8d08 100644 --- a/integ-test/src/test/resources/big5/queries/range_field_conjunction_small_range_small_term_query.ppl +++ b/integ-test/src/test/resources/big5/queries/range_field_conjunction_small_range_small_term_query.ppl @@ -1,4 +1,31 @@ +/* +{ + "name": "range_field_conjunction_small_range_small_term_query", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "bool": { + "should": [ + { + "term": { + "aws.cloudwatch.log_stream": "indigodagger" + } + }, + { + "range": { + "metrics.size": { + "gte": 10, + "lte": 20 + } + } + } + ] + } + } + } +} +*/ source = big5 -| where `aws.cloudwatch.log_stream` = 'indigodagger' - or (`metrics.size` >= 1 and `metrics.size` <= 30) +| where `aws.cloudwatch.log_stream` = 'indigodagger' or (`metrics.size` >= 10 and `metrics.size` <= 20) | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_field_disjunction_big_range_small_term_query.ppl b/integ-test/src/test/resources/big5/queries/range_field_disjunction_big_range_small_term_query.ppl index 4ea1dcfc518..59ac3769159 100644 --- a/integ-test/src/test/resources/big5/queries/range_field_disjunction_big_range_small_term_query.ppl +++ b/integ-test/src/test/resources/big5/queries/range_field_disjunction_big_range_small_term_query.ppl @@ -1,4 +1,31 @@ +/* +{ + "name": "range_field_disjunction_big_range_small_term_query", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "bool": { + "should": [ + { + "term": { + "aws.cloudwatch.log_stream": "indigodagger" + } + }, + { + "range": { + "metrics.size": { + "gte": 1, + "lte": 100 + } + } + } + ] + } + } + } +} +*/ source = big5 -| where `aws.cloudwatch.log_stream` = 'indigodagger' - or (`metrics.size` >= 1 and `metrics.size` <= 1000) +| where `aws.cloudwatch.log_stream` = 'indigodagger' or (`metrics.size` >= 1 and `metrics.size` <= 100) | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_numeric.ppl b/integ-test/src/test/resources/big5/queries/range_numeric.ppl index 5b5b50b7c35..dfdabbf0877 100644 --- a/integ-test/src/test/resources/big5/queries/range_numeric.ppl +++ b/integ-test/src/test/resources/big5/queries/range_numeric.ppl @@ -1,3 +1,20 @@ +/* +{ + "name": "range-numeric", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "range": { + "metrics.size": { + "gte": 20, + "lte": 200 + } + } + } + } +} +*/ source = big5 -| where `metrics.size` >= 1 and `metrics.size` <= 1000 +| where `metrics.size` >= 20 and `metrics.size` <= 200 | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_with_asc_sort.ppl b/integ-test/src/test/resources/big5/queries/range_with_asc_sort.ppl index a3325df54ed..f9ef584a340 100644 --- a/integ-test/src/test/resources/big5/queries/range_with_asc_sort.ppl +++ b/integ-test/src/test/resources/big5/queries/range_with_asc_sort.ppl @@ -1,5 +1,24 @@ +/* +{ + "name": "range_with_asc_sort", + "operation-type": "search", + "index": "{{ index_name | default('big5') }}", + "body": { + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-01T00:00:00", + "lte": "2023-01-13T00:00:00" + } + } + }, + "sort": [ + { "@timestamp": "asc" } + ] + } +} +*/ source = big5 -| where `@timestamp` >= '2023-01-01 00:00:00' - and `@timestamp` <= '2023-01-13 00:00:00' +| where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` <= '2023-01-13 00:00:00' | sort + `@timestamp` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_with_desc_sort.ppl b/integ-test/src/test/resources/big5/queries/range_with_desc_sort.ppl index ba3a042d511..e98fc75acc0 100644 --- a/integ-test/src/test/resources/big5/queries/range_with_desc_sort.ppl +++ b/integ-test/src/test/resources/big5/queries/range_with_desc_sort.ppl @@ -1,5 +1,24 @@ +/* +{ + "name": "range_with_desc_sort", + "operation-type": "search", + "index": "{{ index_name | default('big5') }}", + "body": { + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-01T00:00:00", + "lte": "2023-01-13T00:00:00" + } + } + }, + "sort": [ + { "@timestamp": "desc" } + ] + } +} +*/ source = big5 -| where `@timestamp` >= '2023-01-01 00:00:00' - and `@timestamp` <= '2023-01-13 00:00:00' +| where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` <= '2023-01-13 00:00:00' | sort - `@timestamp` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/scroll.ppl b/integ-test/src/test/resources/big5/queries/scroll.ppl index 6b63c414ac0..9bb35aaf830 100644 --- a/integ-test/src/test/resources/big5/queries/scroll.ppl +++ b/integ-test/src/test/resources/big5/queries/scroll.ppl @@ -1,2 +1,17 @@ +/* +{ + "name": "scroll", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "pages": 25, + "results-per-page": 1000, + "body": { + "query": { + "match_all": {} + } + } +} +*/ +/* scroll is unsupported in PPL */ source = big5 | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/sort_keyword_can_match_shortcut.ppl b/integ-test/src/test/resources/big5/queries/sort_keyword_can_match_shortcut.ppl index aab85fb7c1b..c2f2d89be18 100644 --- a/integ-test/src/test/resources/big5/queries/sort_keyword_can_match_shortcut.ppl +++ b/integ-test/src/test/resources/big5/queries/sort_keyword_can_match_shortcut.ppl @@ -1,3 +1,21 @@ +/* +{ + "name": "sort_keyword_can_match_shortcut", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "track_total_hits": false, + "query": { + "match": { + "process.name": "kernel" + } + }, + "sort" : [ + {"meta.file" : "asc"} + ] + } +} +*/ source = big5 process.name=kernel -| sort + `@timestamp` +| sort + `meta.file` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/sort_keyword_no_can_match_shortcut.ppl b/integ-test/src/test/resources/big5/queries/sort_keyword_no_can_match_shortcut.ppl index aab85fb7c1b..de375ef4517 100644 --- a/integ-test/src/test/resources/big5/queries/sort_keyword_no_can_match_shortcut.ppl +++ b/integ-test/src/test/resources/big5/queries/sort_keyword_no_can_match_shortcut.ppl @@ -1,3 +1,24 @@ +/* +{ + "name": "sort_keyword_no_can_match_shortcut", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "request-params" : { + "pre_filter_shard_size" : 100000 + }, + "body": { + "track_total_hits": false, + "query": { + "match": { + "process.name": "kernel" + } + }, + "sort" : [ + {"meta.file" : "asc"} + ] + } +} +*/ source = big5 process.name=kernel -| sort + `@timestamp` +| sort + `meta.file` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/sort_numeric_asc.ppl b/integ-test/src/test/resources/big5/queries/sort_numeric_asc.ppl index eb96d2b4bab..6340dfa854e 100644 --- a/integ-test/src/test/resources/big5/queries/sort_numeric_asc.ppl +++ b/integ-test/src/test/resources/big5/queries/sort_numeric_asc.ppl @@ -1,3 +1,21 @@ +/* +{ + "name": "sort_numeric_asc", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "track_total_hits": false, + "query": { + "match_all": {} + }, + "sort": [ + { + "metrics.size": "asc" + } + ] + } +} +*/ source = big5 | sort + `metrics.size` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/sort_numeric_asc_with_match.ppl b/integ-test/src/test/resources/big5/queries/sort_numeric_asc_with_match.ppl index 198667db866..417bc4dcf46 100644 --- a/integ-test/src/test/resources/big5/queries/sort_numeric_asc_with_match.ppl +++ b/integ-test/src/test/resources/big5/queries/sort_numeric_asc_with_match.ppl @@ -1,3 +1,23 @@ +/* +{ + "name": "sort_numeric_asc_with_match", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "track_total_hits": false, + "query": { + "match": { + "log.file.path": "/var/log/messages/solarshark" + } + }, + "sort": [ + { + "metrics.size": "asc" + } + ] + } +} +*/ source = big5 log.file.path=\"/var/log/messages/solarshark\" | sort + `metrics.size` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/sort_numeric_desc.ppl b/integ-test/src/test/resources/big5/queries/sort_numeric_desc.ppl index f4a4165fbfc..b55d15a135e 100644 --- a/integ-test/src/test/resources/big5/queries/sort_numeric_desc.ppl +++ b/integ-test/src/test/resources/big5/queries/sort_numeric_desc.ppl @@ -1,3 +1,21 @@ +/* +{ + "name": "sort_numeric_desc", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "track_total_hits": false, + "query": { + "match_all": {} + }, + "sort": [ + { + "metrics.size": "desc" + } + ] + } +} +*/ source = big5 | sort - `metrics.size` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/sort_numeric_desc_with_match.ppl b/integ-test/src/test/resources/big5/queries/sort_numeric_desc_with_match.ppl index f282e9ae67d..53ece3b052b 100644 --- a/integ-test/src/test/resources/big5/queries/sort_numeric_desc_with_match.ppl +++ b/integ-test/src/test/resources/big5/queries/sort_numeric_desc_with_match.ppl @@ -1,3 +1,23 @@ +/* +{ + "name": "sort_numeric_desc_with_match", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "track_total_hits": false, + "query": { + "match": { + "log.file.path": "/var/log/messages/solarshark" + } + }, + "sort": [ + { + "metrics.size": "desc" + } + ] + } +} +*/ source = big5 log.file.path=\"/var/log/messages/solarshark\" | sort - `metrics.size` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/term.ppl b/integ-test/src/test/resources/big5/queries/term.ppl index 2cbfae69eba..20799833371 100644 --- a/integ-test/src/test/resources/big5/queries/term.ppl +++ b/integ-test/src/test/resources/big5/queries/term.ppl @@ -1,3 +1,20 @@ +/* +{ + "name": "term", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "request-timeout": 7200, + "body": { + "query": { + "term": { + "log.file.path": { + "value": "/var/log/messages/birdknight" + } + } + } + } +} +*/ source = big5 | where `log.file.path` = '/var/log/messages/birdknight' | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/terms_significant_1.ppl b/integ-test/src/test/resources/big5/queries/terms_significant_1.ppl index b33048f82a1..452e0f23ae5 100644 --- a/integ-test/src/test/resources/big5/queries/terms_significant_1.ppl +++ b/integ-test/src/test/resources/big5/queries/terms_significant_1.ppl @@ -1,5 +1,41 @@ +/* +{ + "name": "terms-significant-1", + "operation-type": "search", + "request-timeout": 7200, + "index": "{{index_name | default('big5')}}", + "body": + { + "track_total_hits": false, + "size": 0, + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-01T00:00:00", + "lt": "2023-01-03T00:00:00" + } + } + }, + "aggs": { + "terms": { + "terms": { + "field": "aws.cloudwatch.log_stream", + "size": 10 + }, + "aggs": { + "significant_ips": { + "significant_terms": { + "field": "process.name" + } + } + } + } + } + } +} +*/ +/* significant_terms is unsupported in PPL */ source = big5 -| where `@timestamp` >= '2023-01-01 00:00:00' - and `@timestamp` < '2023-01-03 00:00:00' -| stats count() by `aws.cloudwatch.log_stream` +| where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' +| stats count() by `aws.cloudwatch.log_stream`, `process.name` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/terms_significant_2.ppl b/integ-test/src/test/resources/big5/queries/terms_significant_2.ppl index 994914e3bbe..954cf194e4d 100644 --- a/integ-test/src/test/resources/big5/queries/terms_significant_2.ppl +++ b/integ-test/src/test/resources/big5/queries/terms_significant_2.ppl @@ -1,5 +1,40 @@ +/* +{ + "name": "terms-significant-2", + "operation-type": "search", + "request-timeout": 7200, + "index": "{{index_name | default('big5')}}", + "body": { + "track_total_hits": false, + "size": 0, + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-01T00:00:00", + "lt": "2023-01-03T00:00:00" + } + } + }, + "aggs": { + "terms": { + "terms": { + "field": "process.name", + "size": 10 + }, + "aggs": { + "significant_ips": { + "significant_terms": { + "field": "aws.cloudwatch.log_stream" + } + } + } + } + } + } +} +*/ +/* significant_terms is unsupported in PPL */ source = big5 -| where `@timestamp` >= '2023-01-01 00:00:00' - and `@timestamp` < '2023-01-03 00:00:00' -| stats count() by `process.name` +| where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' +| stats count() by `process.name`, `aws.cloudwatch.log_stream` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/all_queries.ppl b/integ-test/src/test/resources/clickbench/all_queries.ppl new file mode 100644 index 00000000000..c47fa7ec450 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/all_queries.ppl @@ -0,0 +1,551 @@ +-- File: q1.ppl +/* +SELECT COUNT(*) FROM hits; +*/ +source=hits | stats count() + +-- File: q10.ppl +/* +SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) +FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10; +*/ +source=hits s +| stats bucket_nullable=false sum(AdvEngineID), count() as c, avg(ResolutionWidth), dc(UserID) by RegionID +| sort - c +| head 10 + + +-- File: q11.ppl +/* +SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u +FROM hits WHERE MobilePhoneModel <> '' +GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10; +*/ +source=hits +| where MobilePhoneModel != '' +| stats bucket_nullable=false dc(UserID) as u by MobilePhoneModel +| sort - u +| head 10 + + +-- File: q12.ppl +/* +SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u +FROM hits WHERE MobilePhoneModel <> '' +GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10; +*/ +source=hits +| where MobilePhoneModel != '' +| stats bucket_nullable=false dc(UserID) as u by MobilePhone, MobilePhoneModel +| sort - u +| head 10 + + +-- File: q13.ppl +/* +SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' +GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +*/ +source=hits +| where SearchPhrase != '' +| stats bucket_nullable=false count() as c by SearchPhrase +| sort - c +| head 10 + + +-- File: q14.ppl +/* +SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u +FROM hits WHERE SearchPhrase <> '' +GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10; +*/ +source=hits +| where SearchPhrase != '' +| stats bucket_nullable=false dc(UserID) as u by SearchPhrase +| sort - u +| head 10 + + +-- File: q15.ppl +/* +SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c +FROM hits WHERE SearchPhrase <> '' +GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10; +*/ +source=hits +| where SearchPhrase != '' +| stats bucket_nullable=false count() as c by SearchEngineID, SearchPhrase +| sort - c +| head 10 + + +-- File: q16.ppl +/* +SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10; +*/ +source=hits +| stats bucket_nullable=false count() by UserID +| sort - `count()` +| head 10 + + +-- File: q17.ppl +/* +SELECT UserID, SearchPhrase, COUNT(*) +FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; +*/ +source=hits +| stats bucket_nullable=false count() by UserID, SearchPhrase +| sort - `count()` +| head 10 + + +-- File: q18.ppl +/* +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10; +*/ +source=hits +| stats bucket_nullable=false count() by UserID, SearchPhrase +| head 10 + + +-- File: q19.ppl +/* +SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) +FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; +*/ +source=hits +| eval m = extract(minute from EventTime) +| stats bucket_nullable=false count() by UserID, m, SearchPhrase +| sort - `count()` +| head 10 + + +-- File: q2.ppl +/* +SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; +*/ +source=hits | where AdvEngineID!=0 | stats count() + +-- File: q20.ppl +/* +SELECT UserID FROM hits WHERE UserID = 435090932899640449; +*/ +source=hits +| where UserID = 435090932899640449 +| fields UserID + +-- File: q21.ppl +/* +SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%'; +*/ +source=hits +| where like(URL, '%google%') +| stats count() + +-- File: q22.ppl +/* +SELECT SearchPhrase, MIN(URL), COUNT(*) AS c +FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' +GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +*/ +source=hits +| where like(URL, '%google%') and SearchPhrase != '' +| stats bucket_nullable=false /* min(URL), */ count() as c by SearchPhrase +| sort - c +| head 10 + + +-- File: q23.ppl +/* +SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) +FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' +GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +*/ +source=hits +| where like(Title, '%Google%') and not like(URL, '%.google.%') and SearchPhrase != '' +| stats bucket_nullable=false /* min(URL), min(Title), */ count() as c, dc(UserID) by SearchPhrase +| sort - c +| head 10 + + +-- File: q24.ppl +/* +SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10; +*/ +source=hits +| where like(URL, '%google%') +| sort EventTime +| head 10 + +-- File: q25.ppl +/* +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10; +*/ +source=hits +| where SearchPhrase != '' +| sort EventTime +| fields SearchPhrase +| head 10 + +-- File: q26.ppl +/* +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10; +*/ +source=hits +| where SearchPhrase != '' +| fields SearchPhrase +| sort SearchPhrase +| head 10 + +-- File: q27.ppl +/* +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10; +*/ +source=hits +| where SearchPhrase != '' +| sort EventTime, SearchPhrase +| fields SearchPhrase +| head 10 + +-- File: q28.ppl +/* +SELECT CounterID, AVG(length(URL)) AS l, COUNT(*) AS c +FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +*/ +source=hits +| where URL != '' +| stats bucket_nullable=false avg(length(URL)) as l, count() as c by CounterID +| where c > 100000 +| sort - l +| head 25 + + +-- File: q29.ppl +/* +SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, +AVG(length(Referer)) AS l, COUNT(*) AS c, MIN(Referer) +FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +*/ +/* +OpenSearch accepts Json as restful request payload, convert \. to \\. and \1 to \\1 +*/ +source=hits +| where Referer != '' +| eval k = regexp_replace(Referer, '^https?://(?:www\\.)?([^/]+)/.*$', '\\1') +| stats bucket_nullable=false avg(length(Referer)) as l, count() as c, min(Referer) by k +| where c > 100000 +| sort - l +| head 25 + + +-- File: q3.ppl +/* +SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits; +*/ +source=hits | stats sum(AdvEngineID), count() , avg(ResolutionWidth) + +-- File: q30.ppl +/* +SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), ... SUM(ResolutionWidth + 89) FROM hits; +*/ +source=hits +| stats + sum(ResolutionWidth), + sum(ResolutionWidth+1), + sum(ResolutionWidth+2), + sum(ResolutionWidth+3), + sum(ResolutionWidth+4), + sum(ResolutionWidth+5), + sum(ResolutionWidth+6), + sum(ResolutionWidth+7), + sum(ResolutionWidth+8), + sum(ResolutionWidth+9), + sum(ResolutionWidth+10), + sum(ResolutionWidth+11), + sum(ResolutionWidth+12), + sum(ResolutionWidth+13), + sum(ResolutionWidth+14), + sum(ResolutionWidth+15), + sum(ResolutionWidth+16), + sum(ResolutionWidth+17), + sum(ResolutionWidth+18), + sum(ResolutionWidth+19), + sum(ResolutionWidth+20), + sum(ResolutionWidth+21), + sum(ResolutionWidth+22), + sum(ResolutionWidth+23), + sum(ResolutionWidth+24), + sum(ResolutionWidth+25), + sum(ResolutionWidth+26), + sum(ResolutionWidth+27), + sum(ResolutionWidth+28), + sum(ResolutionWidth+29), + sum(ResolutionWidth+30), + sum(ResolutionWidth+31), + sum(ResolutionWidth+32), + sum(ResolutionWidth+33), + sum(ResolutionWidth+34), + sum(ResolutionWidth+35), + sum(ResolutionWidth+36), + sum(ResolutionWidth+37), + sum(ResolutionWidth+38), + sum(ResolutionWidth+39), + sum(ResolutionWidth+40), + sum(ResolutionWidth+41), + sum(ResolutionWidth+42), + sum(ResolutionWidth+43), + sum(ResolutionWidth+44), + sum(ResolutionWidth+45), + sum(ResolutionWidth+46), + sum(ResolutionWidth+47), + sum(ResolutionWidth+48), + sum(ResolutionWidth+49), + sum(ResolutionWidth+50), + sum(ResolutionWidth+51), + sum(ResolutionWidth+52), + sum(ResolutionWidth+53), + sum(ResolutionWidth+54), + sum(ResolutionWidth+55), + sum(ResolutionWidth+56), + sum(ResolutionWidth+57), + sum(ResolutionWidth+58), + sum(ResolutionWidth+59), + sum(ResolutionWidth+60), + sum(ResolutionWidth+61), + sum(ResolutionWidth+62), + sum(ResolutionWidth+63), + sum(ResolutionWidth+64), + sum(ResolutionWidth+65), + sum(ResolutionWidth+66), + sum(ResolutionWidth+67), + sum(ResolutionWidth+68), + sum(ResolutionWidth+69), + sum(ResolutionWidth+70), + sum(ResolutionWidth+71), + sum(ResolutionWidth+72), + sum(ResolutionWidth+73), + sum(ResolutionWidth+74), + sum(ResolutionWidth+75), + sum(ResolutionWidth+76), + sum(ResolutionWidth+77), + sum(ResolutionWidth+78), + sum(ResolutionWidth+79), + sum(ResolutionWidth+80), + sum(ResolutionWidth+81), + sum(ResolutionWidth+82), + sum(ResolutionWidth+83), + sum(ResolutionWidth+84), + sum(ResolutionWidth+85), + sum(ResolutionWidth+86), + sum(ResolutionWidth+87), + sum(ResolutionWidth+88), + sum(ResolutionWidth+89) + +-- File: q31.ppl +/* +SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) +FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10; +*/ +source=hits +| where SearchPhrase != '' +| stats bucket_nullable=false count() as c, sum(IsRefresh), avg(ResolutionWidth) by SearchEngineID, ClientIP +| sort - c +| head 10 + + +-- File: q32.ppl +/* +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) +FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; +*/ +source=hits +| where SearchPhrase != '' +| stats bucket_nullable=false count() as c, sum(IsRefresh), avg(ResolutionWidth) by WatchID, ClientIP +| sort - c +| head 10 + + +-- File: q33.ppl +/* +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) +FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; +*/ +source=hits +| stats bucket_nullable=false count() as c, sum(IsRefresh), avg(ResolutionWidth) by WatchID, ClientIP +| sort - c +| head 10 + + +-- File: q34.ppl +/* +SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10; +*/ +source=hits +| stats bucket_nullable=false count() as c by URL +| sort - c +| head 10 + + +-- File: q35.ppl +/* +SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10; +*/ +source=hits +| eval const = 1 +| stats bucket_nullable=false count() as c by const, URL +| sort - c +| head 10 + + +-- File: q36.ppl +/* +SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c +FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10; +*/ +source=hits +| eval `ClientIP - 1` = ClientIP - 1, `ClientIP - 2` = ClientIP - 2, `ClientIP - 3` = ClientIP - 3 +| stats bucket_nullable=false count() as c by `ClientIP`, `ClientIP - 1`, `ClientIP - 2`, `ClientIP - 3` +| sort - c +| head 10 + + +-- File: q37.ppl +/* +SELECT URL, COUNT(*) AS PageViews +FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' +AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' +GROUP BY URL ORDER BY PageViews DESC LIMIT 10; +*/ +source=hits +| where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and DontCountHits = 0 and IsRefresh = 0 and URL != '' +| stats bucket_nullable=false count() as PageViews by URL +| sort - PageViews +| head 10 + + +-- File: q38.ppl +/* +SELECT Title, COUNT(*) AS PageViews +FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' +AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' +GROUP BY Title ORDER BY PageViews DESC LIMIT 10; +*/ +source=hits +| where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and DontCountHits = 0 and IsRefresh = 0 and Title != '' +| stats bucket_nullable=false count() as PageViews by Title +| sort - PageViews +| head 10 + + +-- File: q39.ppl +/* +SELECT URL, COUNT(*) AS PageViews +FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' +AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 +GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +*/ +source=hits +| where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and IsRefresh = 0 and IsLink != 0 and IsDownload = 0 +| stats bucket_nullable=false count() as PageViews by URL +| sort - PageViews +| head 10 from 1000 + + +-- File: q4.ppl +/* +SELECT AVG(UserID) FROM hits; +*/ +source=hits | stats avg(UserID) + +-- File: q40.ppl +/* +SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews +FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 +GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +*/ +source=hits +| where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and IsRefresh = 0 +| eval Src=case(SearchEngineID = 0 and AdvEngineID = 0, Referer else ''), Dst=URL +| stats bucket_nullable=false count() as PageViews by TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst +| sort - PageViews +| head 10 from 1000 + + +-- File: q41.ppl +/* +SELECT URLHash, EventDate, COUNT(*) AS PageViews +FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' +AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 +GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100; +*/ +source=hits +| where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and IsRefresh = 0 and TraficSourceID in (-1, 6) and RefererHash = 3594120000172545465 +| stats bucket_nullable=false count() as PageViews by URLHash, EventDate +| sort - PageViews +| head 10 from 100 + + +-- File: q42.ppl +/* +SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews +FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' +AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 +GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000; +*/ +source=hits +| where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and IsRefresh = 0 and DontCountHits = 0 and URLHash = 2868770270353813622 +| stats bucket_nullable=false count() as PageViews by WindowClientWidth, WindowClientHeight +| sort - PageViews +| head 10 from 10000 + + +-- File: q43.ppl +/* +SELECT DATE_FORMAT(EventTime, '%Y-%m-%d %H:00:00') AS M, COUNT(*) AS PageViews +FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' +AND IsRefresh = 0 AND DontCountHits = 0 +GROUP BY DATE_FORMAT(EventTime, '%Y-%m-%d %H:00:00') +ORDER BY DATE_FORMAT(EventTime, '%Y-%m-%d %H:00:00') +LIMIT 10 OFFSET 1000; +*/ +source=hits +| where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-15 00:00:00' and IsRefresh = 0 and DontCountHits = 0 +| eval M = date_format(EventTime, '%Y-%m-%d %H:00:00') +| stats bucket_nullable=false count() as PageViews by M +| sort M +| head 10 from 1000 + + +-- File: q5.ppl +/* +SELECT COUNT(DISTINCT UserID) FROM hits; +*/ +source=hits | stats dc(UserID) + +-- File: q6.ppl +/* +SELECT COUNT(DISTINCT SearchPhrase) FROM hits; +*/ +source=hits | stats dc(SearchPhrase) + +-- File: q7.ppl +/* +SELECT MIN(EventDate), MAX(EventDate) FROM hits; +*/ +source=hits | stats min(EventDate), max(EventDate) + +-- File: q8.ppl +/* +SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC; +*/ +source=hits | where AdvEngineID!=0 | stats bucket_nullable=false count() by AdvEngineID | sort - `count()` + + +-- File: q9.ppl +/* +SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10; +*/ +source=hits | stats bucket_nullable=false dc(UserID) as u by RegionID | sort -u | head 10 + + diff --git a/integ-test/src/test/resources/clickbench/data/clickbench.json b/integ-test/src/test/resources/clickbench/data/clickbench.json index 7f21bad191f..d3de2554805 100644 --- a/integ-test/src/test/resources/clickbench/data/clickbench.json +++ b/integ-test/src/test/resources/clickbench/data/clickbench.json @@ -1,2 +1,18 @@ -{"index":{}} -{"WatchID":"9110818468285196899","JavaEnable":0,"Title":"","GoodEvent":1,"EventTime":"2013-07-14 20:38:47","EventDate":"2013-07-15","CounterID":17,"ClientIP":-1216690514,"RegionID":839,"UserID":"-2461439046089301801","CounterClass":0,"OS":0,"UserAgent":0,"URL":"","Referer":"https://example.org/about","IsRefresh":0,"RefererCategoryID":0,"RefererRegionID":0,"URLCategoryID":0,"URLRegionID":0,"ResolutionWidth":0,"ResolutionHeight":0,"ResolutionDepth":0,"FlashMajor":0,"FlashMinor":0,"FlashMinor2":"","NetMajor":0,"NetMinor":0,"UserAgentMajor":0,"UserAgentMinor":"�O","CookieEnable":0,"JavascriptEnable":0,"IsMobile":0,"MobilePhone":0,"MobilePhoneModel":"","Params":"","IPNetworkID":3793327,"TraficSourceID":4,"SearchEngineID":0,"SearchPhrase":"","AdvEngineID":0,"IsArtifical":0,"WindowClientWidth":0,"WindowClientHeight":0,"ClientTimeZone":-1,"ClientEventTime":"1971-01-01 14:16:06","SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"PageCharset":"","CodeVersion":0,"IsLink":0,"IsDownload":0,"IsNotBounce":0,"FUniqID":"0","OriginalURL":"","HID":0,"IsOldCounter":0,"IsEvent":0,"IsParameter":0,"DontCountHits":0,"WithHash":0,"HitColor":"5","LocalEventTime":"2013-07-15 10:47:34","Age":0,"Sex":0,"Income":0,"Interests":0,"Robotness":0,"RemoteIP":-1001831330,"WindowName":-1,"OpenerName":-1,"HistoryLength":-1,"BrowserLanguage":"�","BrowserCountry":"�\f","SocialNetwork":"","SocialAction":"","HTTPError":0,"SendTiming":0,"DNSTiming":0,"ConnectTiming":0,"ResponseStartTiming":0,"ResponseEndTiming":0,"FetchTiming":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","ParamPrice":"0","ParamOrderID":"","ParamCurrency":"NH\u001C","ParamCurrencyID":0,"OpenstatServiceName":"","OpenstatCampaignID":"","OpenstatAdID":"","OpenstatSourceID":"","UTMSource":"","UTMMedium":"","UTMCampaign":"","UTMContent":"","UTMTerm":"","FromTag":"","HasGCLID":0,"RefererHash":"-296158784638538920","URLHash":"-8417682003818480435","CLID":0} +{"index": {}} +{"WatchID": "1000000000000000001", "JavaEnable": 1, "Title": "Search Results", "GoodEvent": 1, "EventTime": "2013-07-15 10:30:00", "EventDate": "2013-07-15", "CounterID": 62, "ClientIP": 1234567890, "RegionID": 100, "UserID": "1000000000000000001", "CounterClass": 1, "OS": 1, "UserAgent": 50, "URL": "https://www.google.com/search?q=test", "Referer": "https://example.com", "IsRefresh": 0, "RefererCategoryID": 10, "RefererRegionID": 100, "URLCategoryID": 20, "URLRegionID": 200, "ResolutionWidth": 1920, "ResolutionHeight": 1080, "ResolutionDepth": 24, "FlashMajor": 11, "FlashMinor": 5, "FlashMinor2": "", "NetMajor": 4, "NetMinor": 0, "UserAgentMajor": 50, "UserAgentMinor": "0", "CookieEnable": 1, "JavascriptEnable": 1, "IsMobile": 0, "MobilePhone": 0, "MobilePhoneModel": "", "Params": "", "IPNetworkID": 1000000, "TraficSourceID": 6, "SearchEngineID": 1, "SearchPhrase": "test query google", "AdvEngineID": 0, "IsArtifical": 0, "WindowClientWidth": 1920, "WindowClientHeight": 1080, "ClientTimeZone": 0, "ClientEventTime": "2013-07-15 10:30:00", "SilverlightVersion1": 0, "SilverlightVersion2": 0, "SilverlightVersion3": 0, "SilverlightVersion4": 0, "PageCharset": "utf-8", "CodeVersion": 1, "IsLink": 1, "IsDownload": 0, "IsNotBounce": 1, "FUniqID": "10000001", "OriginalURL": "", "HID": 100001, "IsOldCounter": 0, "IsEvent": 1, "IsParameter": 0, "DontCountHits": 0, "WithHash": 0, "HitColor": "1", "LocalEventTime": "2013-07-15 10:30:00", "Age": 30, "Sex": 1, "Income": 3, "Interests": 100, "Robotness": 0, "RemoteIP": 1234567890, "WindowName": 100, "OpenerName": 200, "HistoryLength": 10, "BrowserLanguage": "en", "BrowserCountry": "US", "SocialNetwork": "", "SocialAction": "", "HTTPError": 0, "SendTiming": 100, "DNSTiming": 50, "ConnectTiming": 100, "ResponseStartTiming": 200, "ResponseEndTiming": 300, "FetchTiming": 400, "SocialSourceNetworkID": 0, "SocialSourcePage": "", "ParamPrice": "0", "ParamOrderID": "", "ParamCurrency": "USD", "ParamCurrencyID": 1, "OpenstatServiceName": "", "OpenstatCampaignID": "", "OpenstatAdID": "", "OpenstatSourceID": "", "UTMSource": "", "UTMMedium": "", "UTMCampaign": "", "UTMContent": "", "UTMTerm": "", "FromTag": "", "HasGCLID": 0, "RefererHash": "3594120000172545465", "URLHash": "1000000000000000001", "CLID": 1000} +{"index": {}} +{"WatchID": "1000000000000000002", "JavaEnable": 1, "Title": "Google Search Page", "GoodEvent": 1, "EventTime": "2013-07-14 14:20:00", "EventDate": "2013-07-14", "CounterID": 62, "ClientIP": 1234567891, "RegionID": 101, "UserID": "1000000000000000002", "CounterClass": 1, "OS": 2, "UserAgent": 51, "URL": "https://google.com/maps", "Referer": "https://example.com", "IsRefresh": 0, "RefererCategoryID": 11, "RefererRegionID": 101, "URLCategoryID": 21, "URLRegionID": 201, "ResolutionWidth": 1366, "ResolutionHeight": 768, "ResolutionDepth": 24, "FlashMajor": 10, "FlashMinor": 4, "FlashMinor2": "", "NetMajor": 3, "NetMinor": 9, "UserAgentMajor": 51, "UserAgentMinor": "0", "CookieEnable": 1, "JavascriptEnable": 1, "IsMobile": 0, "MobilePhone": 0, "MobilePhoneModel": "", "Params": "", "IPNetworkID": 1000001, "TraficSourceID": -1, "SearchEngineID": 2, "SearchPhrase": "search something", "AdvEngineID": 0, "IsArtifical": 0, "WindowClientWidth": 1366, "WindowClientHeight": 768, "ClientTimeZone": -5, "ClientEventTime": "2013-07-14 14:20:00", "SilverlightVersion1": 0, "SilverlightVersion2": 0, "SilverlightVersion3": 0, "SilverlightVersion4": 0, "PageCharset": "utf-8", "CodeVersion": 1, "IsLink": 0, "IsDownload": 0, "IsNotBounce": 1, "FUniqID": "10000002", "OriginalURL": "", "HID": 100002, "IsOldCounter": 0, "IsEvent": 0, "IsParameter": 0, "DontCountHits": 0, "WithHash": 0, "HitColor": "2", "LocalEventTime": "2013-07-14 14:20:00", "Age": 25, "Sex": 2, "Income": 2, "Interests": 200, "Robotness": 0, "RemoteIP": 1234567891, "WindowName": 101, "OpenerName": 201, "HistoryLength": 5, "BrowserLanguage": "en", "BrowserCountry": "UK", "SocialNetwork": "", "SocialAction": "", "HTTPError": 0, "SendTiming": 110, "DNSTiming": 55, "ConnectTiming": 105, "ResponseStartTiming": 205, "ResponseEndTiming": 305, "FetchTiming": 405, "SocialSourceNetworkID": 0, "SocialSourcePage": "", "ParamPrice": "0", "ParamOrderID": "", "ParamCurrency": "GBP", "ParamCurrencyID": 3, "OpenstatServiceName": "", "OpenstatCampaignID": "", "OpenstatAdID": "", "OpenstatSourceID": "", "UTMSource": "", "UTMMedium": "", "UTMCampaign": "", "UTMContent": "", "UTMTerm": "", "FromTag": "", "HasGCLID": 0, "RefererHash": "3594120000172545465", "URLHash": "1000000000000000002", "CLID": 1001} +{"index": {}} +{"WatchID": "1000000000000000003", "JavaEnable": 0, "Title": "Welcome to Google", "GoodEvent": 1, "EventTime": "2013-07-10 09:15:00", "EventDate": "2013-07-10", "CounterID": 62, "ClientIP": 1234567892, "RegionID": 102, "UserID": "1000000000000000003", "CounterClass": 0, "OS": 3, "UserAgent": 52, "URL": "https://example.com/page?ref=google", "Referer": "https://www.google.com", "IsRefresh": 0, "RefererCategoryID": 12, "RefererRegionID": 102, "URLCategoryID": 22, "URLRegionID": 202, "ResolutionWidth": 1280, "ResolutionHeight": 1024, "ResolutionDepth": 32, "FlashMajor": 9, "FlashMinor": 3, "FlashMinor2": "", "NetMajor": 2, "NetMinor": 8, "UserAgentMajor": 52, "UserAgentMinor": "0", "CookieEnable": 0, "JavascriptEnable": 1, "IsMobile": 1, "MobilePhone": 1, "MobilePhoneModel": "iPhone", "Params": "", "IPNetworkID": 1000002, "TraficSourceID": 6, "SearchEngineID": 1, "SearchPhrase": "google search", "AdvEngineID": 1, "IsArtifical": 0, "WindowClientWidth": 1280, "WindowClientHeight": 1024, "ClientTimeZone": 5, "ClientEventTime": "2013-07-10 09:15:00", "SilverlightVersion1": 0, "SilverlightVersion2": 0, "SilverlightVersion3": 0, "SilverlightVersion4": 0, "PageCharset": "utf-8", "CodeVersion": 2, "IsLink": 1, "IsDownload": 0, "IsNotBounce": 0, "FUniqID": "10000003", "OriginalURL": "", "HID": 100003, "IsOldCounter": 1, "IsEvent": 1, "IsParameter": 1, "DontCountHits": 0, "WithHash": 1, "HitColor": "0", "LocalEventTime": "2013-07-10 09:15:00", "Age": 35, "Sex": 1, "Income": 4, "Interests": 300, "Robotness": 1, "RemoteIP": 1234567892, "WindowName": 102, "OpenerName": 202, "HistoryLength": 15, "BrowserLanguage": "en", "BrowserCountry": "US", "SocialNetwork": "", "SocialAction": "", "HTTPError": 0, "SendTiming": 120, "DNSTiming": 60, "ConnectTiming": 110, "ResponseStartTiming": 210, "ResponseEndTiming": 310, "FetchTiming": 410, "SocialSourceNetworkID": 0, "SocialSourcePage": "", "ParamPrice": "10.99", "ParamOrderID": "ORDER001", "ParamCurrency": "USD", "ParamCurrencyID": 1, "OpenstatServiceName": "", "OpenstatCampaignID": "", "OpenstatAdID": "", "OpenstatSourceID": "", "UTMSource": "google", "UTMMedium": "organic", "UTMCampaign": "", "UTMContent": "", "UTMTerm": "", "FromTag": "", "HasGCLID": 1, "RefererHash": "1000000000000000003", "URLHash": "2868770270353813622", "CLID": 1002} +{"index": {}} +{"WatchID": "1000000000000000004", "JavaEnable": 1, "Title": "Google Homepage", "GoodEvent": 0, "EventTime": "2013-07-02 16:45:00", "EventDate": "2013-07-02", "CounterID": 62, "ClientIP": 1234567893, "RegionID": 103, "UserID": "1000000000000000004", "CounterClass": 1, "OS": 4, "UserAgent": 53, "URL": "https://www.google.example.com/test", "Referer": "", "IsRefresh": 0, "RefererCategoryID": 13, "RefererRegionID": 103, "URLCategoryID": 23, "URLRegionID": 203, "ResolutionWidth": 1024, "ResolutionHeight": 768, "ResolutionDepth": 24, "FlashMajor": 11, "FlashMinor": 2, "FlashMinor2": "", "NetMajor": 4, "NetMinor": 1, "UserAgentMajor": 53, "UserAgentMinor": "0", "CookieEnable": 1, "JavascriptEnable": 0, "IsMobile": 0, "MobilePhone": 0, "MobilePhoneModel": "", "Params": "", "IPNetworkID": 1000003, "TraficSourceID": -1, "SearchEngineID": 3, "SearchPhrase": "test", "AdvEngineID": 2, "IsArtifical": 1, "WindowClientWidth": 1024, "WindowClientHeight": 768, "ClientTimeZone": 0, "ClientEventTime": "2013-07-02 16:45:00", "SilverlightVersion1": 0, "SilverlightVersion2": 0, "SilverlightVersion3": 0, "SilverlightVersion4": 0, "PageCharset": "utf-8", "CodeVersion": 3, "IsLink": 0, "IsDownload": 0, "IsNotBounce": 1, "FUniqID": "10000004", "OriginalURL": "", "HID": 100004, "IsOldCounter": 0, "IsEvent": 0, "IsParameter": 0, "DontCountHits": 0, "WithHash": 0, "HitColor": "5", "LocalEventTime": "2013-07-02 16:45:00", "Age": 40, "Sex": 2, "Income": 5, "Interests": 400, "Robotness": 0, "RemoteIP": 1234567893, "WindowName": 103, "OpenerName": 203, "HistoryLength": 20, "BrowserLanguage": "fr", "BrowserCountry": "FR", "SocialNetwork": "", "SocialAction": "", "HTTPError": 1, "SendTiming": 130, "DNSTiming": 65, "ConnectTiming": 115, "ResponseStartTiming": 215, "ResponseEndTiming": 315, "FetchTiming": 415, "SocialSourceNetworkID": 0, "SocialSourcePage": "", "ParamPrice": "25.50", "ParamOrderID": "", "ParamCurrency": "EUR", "ParamCurrencyID": 5, "OpenstatServiceName": "", "OpenstatCampaignID": "", "OpenstatAdID": "", "OpenstatSourceID": "", "UTMSource": "", "UTMMedium": "", "UTMCampaign": "", "UTMContent": "", "UTMTerm": "", "FromTag": "", "HasGCLID": 0, "RefererHash": "1000000000000000004", "URLHash": "2868770270353813622", "CLID": 1003} +{"index": {}} +{"WatchID": "1000000000000000005", "JavaEnable": 1, "Title": "Page with Google Reference", "GoodEvent": 1, "EventTime": "2013-07-05 11:30:00", "EventDate": "2013-07-05", "CounterID": 62, "ClientIP": 1234567894, "RegionID": 104, "UserID": "1000000000000000005", "CounterClass": 0, "OS": 5, "UserAgent": 54, "URL": "https://example.google.org/page", "Referer": "https://google.com", "IsRefresh": 0, "RefererCategoryID": 14, "RefererRegionID": 104, "URLCategoryID": 24, "URLRegionID": 204, "ResolutionWidth": 1920, "ResolutionHeight": 1200, "ResolutionDepth": 32, "FlashMajor": 10, "FlashMinor": 1, "FlashMinor2": "", "NetMajor": 3, "NetMinor": 7, "UserAgentMajor": 54, "UserAgentMinor": "0", "CookieEnable": 1, "JavascriptEnable": 1, "IsMobile": 1, "MobilePhone": 0, "MobilePhoneModel": "Samsung", "Params": "", "IPNetworkID": 1000004, "TraficSourceID": 6, "SearchEngineID": 1, "SearchPhrase": "find something", "AdvEngineID": 0, "IsArtifical": 0, "WindowClientWidth": 1920, "WindowClientHeight": 1200, "ClientTimeZone": 8, "ClientEventTime": "2013-07-05 11:30:00", "SilverlightVersion1": 0, "SilverlightVersion2": 0, "SilverlightVersion3": 0, "SilverlightVersion4": 0, "PageCharset": "utf-8", "CodeVersion": 4, "IsLink": 1, "IsDownload": 1, "IsNotBounce": 1, "FUniqID": "10000005", "OriginalURL": "", "HID": 100005, "IsOldCounter": 1, "IsEvent": 1, "IsParameter": 1, "DontCountHits": 0, "WithHash": 1, "HitColor": "1", "LocalEventTime": "2013-07-05 11:30:00", "Age": 28, "Sex": 1, "Income": 3, "Interests": 500, "Robotness": 1, "RemoteIP": 1234567894, "WindowName": 104, "OpenerName": 204, "HistoryLength": 8, "BrowserLanguage": "de", "BrowserCountry": "DE", "SocialNetwork": "", "SocialAction": "", "HTTPError": 0, "SendTiming": 140, "DNSTiming": 70, "ConnectTiming": 120, "ResponseStartTiming": 220, "ResponseEndTiming": 320, "FetchTiming": 420, "SocialSourceNetworkID": 0, "SocialSourcePage": "", "ParamPrice": "100.00", "ParamOrderID": "ORDER002", "ParamCurrency": "EUR", "ParamCurrencyID": 5, "OpenstatServiceName": "", "OpenstatCampaignID": "", "OpenstatAdID": "", "OpenstatSourceID": "", "UTMSource": "", "UTMMedium": "", "UTMCampaign": "", "UTMContent": "", "UTMTerm": "", "FromTag": "", "HasGCLID": 0, "RefererHash": "3594120000172545465", "URLHash": "1000000000000000005", "CLID": 1004} +{"index": {}} +{"WatchID": "2203335828757764551", "JavaEnable": 1, "Title": "Blog Post", "GoodEvent": 0, "EventTime": "2013-07-22 21:58:40", "EventDate": "2013-07-22", "CounterID": 54, "ClientIP": 2092362178, "RegionID": 104, "UserID": "435090932899640449", "CounterClass": 1, "OS": 0, "UserAgent": 53, "URL": "https://example.com/search", "Referer": "", "IsRefresh": 1, "RefererCategoryID": 7, "RefererRegionID": 157, "URLCategoryID": 50, "URLRegionID": 970, "ResolutionWidth": 1920, "ResolutionHeight": 768, "ResolutionDepth": 32, "FlashMajor": 7, "FlashMinor": 3, "FlashMinor2": "", "NetMajor": 3, "NetMinor": 8, "UserAgentMajor": 45, "UserAgentMinor": "1", "CookieEnable": 1, "JavascriptEnable": 1, "IsMobile": 1, "MobilePhone": 0, "MobilePhoneModel": "", "Params": "", "IPNetworkID": 5008174, "TraficSourceID": 5, "SearchEngineID": 5, "SearchPhrase": "", "AdvEngineID": 0, "IsArtifical": 1, "WindowClientWidth": 1366, "WindowClientHeight": 0, "ClientTimeZone": -7, "ClientEventTime": "2013-07-21 08:58:40", "SilverlightVersion1": 2, "SilverlightVersion2": 7, "SilverlightVersion3": 1, "SilverlightVersion4": 0, "PageCharset": "", "CodeVersion": 4, "IsLink": 1, "IsDownload": 1, "IsNotBounce": 1, "FUniqID": "97214713", "OriginalURL": "", "HID": 719559, "IsOldCounter": 0, "IsEvent": 1, "IsParameter": 1, "DontCountHits": 1, "WithHash": 0, "HitColor": "1", "LocalEventTime": "2013-07-22 21:58:40", "Age": 24, "Sex": 1, "Income": 4, "Interests": 930, "Robotness": 1, "RemoteIP": -1308347026, "WindowName": 467467, "OpenerName": -224954, "HistoryLength": 80, "BrowserLanguage": "en", "BrowserCountry": "US", "SocialNetwork": "twitter", "SocialAction": "comment", "HTTPError": 0, "SendTiming": 1805, "DNSTiming": 581, "ConnectTiming": 769, "ResponseStartTiming": 1757, "ResponseEndTiming": 1378, "FetchTiming": 3685, "SocialSourceNetworkID": 1, "SocialSourcePage": "", "ParamPrice": "25.50", "ParamOrderID": "", "ParamCurrency": "EUR", "ParamCurrencyID": 3, "OpenstatServiceName": "", "OpenstatCampaignID": "", "OpenstatAdID": "", "OpenstatSourceID": "", "UTMSource": "facebook", "UTMMedium": "", "UTMCampaign": "product_launch", "UTMContent": "", "UTMTerm": "", "FromTag": "", "HasGCLID": 0, "RefererHash": "5432268835925590023", "URLHash": "591582412957927770", "CLID": 78278} +{"index": {}} +{"WatchID": "2310139442600589659", "JavaEnable": 0, "Title": "", "GoodEvent": 1, "EventTime": "2013-07-03 22:50:52", "EventDate": "2013-07-03", "CounterID": 49, "ClientIP": -902092651, "RegionID": 331, "UserID": "6515901508915708000", "CounterClass": 1, "OS": 0, "UserAgent": 94, "URL": "https://example.com/about", "Referer": "https://example.org/about", "IsRefresh": 0, "RefererCategoryID": 5, "RefererRegionID": 785, "URLCategoryID": 45, "URLRegionID": 467, "ResolutionWidth": 1920, "ResolutionHeight": 768, "ResolutionDepth": 0, "FlashMajor": 10, "FlashMinor": 4, "FlashMinor2": "", "NetMajor": 2, "NetMinor": 2, "UserAgentMajor": 37, "UserAgentMinor": "\ufffdO", "CookieEnable": 0, "JavascriptEnable": 0, "IsMobile": 0, "MobilePhone": 1, "MobilePhoneModel": "Samsung", "Params": "", "IPNetworkID": 7355061, "TraficSourceID": 4, "SearchEngineID": 1, "SearchPhrase": "", "AdvEngineID": 2, "IsArtifical": 0, "WindowClientWidth": 1366, "WindowClientHeight": 0, "ClientTimeZone": 12, "ClientEventTime": "2013-07-03 07:50:52", "SilverlightVersion1": 5, "SilverlightVersion2": 3, "SilverlightVersion3": 1, "SilverlightVersion4": 3, "PageCharset": "", "CodeVersion": 5, "IsLink": 1, "IsDownload": 1, "IsNotBounce": 0, "FUniqID": "58010050", "OriginalURL": "", "HID": 907514, "IsOldCounter": 1, "IsEvent": 1, "IsParameter": 0, "DontCountHits": 1, "WithHash": 0, "HitColor": "2", "LocalEventTime": "2013-07-03 22:50:52", "Age": 15, "Sex": 1, "Income": 2, "Interests": 154, "Robotness": 1, "RemoteIP": 1065344496, "WindowName": 728990, "OpenerName": -279689, "HistoryLength": 37, "BrowserLanguage": "fr", "BrowserCountry": "DE", "SocialNetwork": "", "SocialAction": "like", "HTTPError": 0, "SendTiming": 1404, "DNSTiming": 572, "ConnectTiming": 42, "ResponseStartTiming": 363, "ResponseEndTiming": 1953, "FetchTiming": 207, "SocialSourceNetworkID": 8, "SocialSourcePage": "", "ParamPrice": "25.50", "ParamOrderID": "992575", "ParamCurrency": "GBP", "ParamCurrencyID": 3, "OpenstatServiceName": "", "OpenstatCampaignID": "", "OpenstatAdID": "", "OpenstatSourceID": "", "UTMSource": "google", "UTMMedium": "social", "UTMCampaign": "", "UTMContent": "", "UTMTerm": "", "FromTag": "", "HasGCLID": 0, "RefererHash": "-8077704597439223167", "URLHash": "-5143777431859549789", "CLID": 3124} +{"index": {}} +{"WatchID": "2133884852119575301", "JavaEnable": 0, "Title": "Blog Post", "GoodEvent": 1, "EventTime": "2013-07-11 05:19:46", "EventDate": "2013-07-11", "CounterID": 13, "ClientIP": 437216268, "RegionID": 479, "UserID": "-6563107575738400561", "CounterClass": 0, "OS": 6, "UserAgent": 66, "URL": "https://example.com/products", "Referer": "https://facebook.com/", "IsRefresh": 1, "RefererCategoryID": 39, "RefererRegionID": 322, "URLCategoryID": 87, "URLRegionID": 641, "ResolutionWidth": 1366, "ResolutionHeight": 1080, "ResolutionDepth": 32, "FlashMajor": 2, "FlashMinor": 3, "FlashMinor2": "", "NetMajor": 3, "NetMinor": 3, "UserAgentMajor": 50, "UserAgentMinor": "2", "CookieEnable": 1, "JavascriptEnable": 0, "IsMobile": 0, "MobilePhone": 0, "MobilePhoneModel": "Samsung", "Params": "", "IPNetworkID": 7165521, "TraficSourceID": 8, "SearchEngineID": 1, "SearchPhrase": "test query", "AdvEngineID": 4, "IsArtifical": 0, "WindowClientWidth": 1024, "WindowClientHeight": 1080, "ClientTimeZone": -3, "ClientEventTime": "2013-07-09 17:19:46", "SilverlightVersion1": 1, "SilverlightVersion2": 3, "SilverlightVersion3": 0, "SilverlightVersion4": 9, "PageCharset": "", "CodeVersion": 5, "IsLink": 0, "IsDownload": 0, "IsNotBounce": 0, "FUniqID": "89555738", "OriginalURL": "", "HID": 528187, "IsOldCounter": 0, "IsEvent": 0, "IsParameter": 1, "DontCountHits": 1, "WithHash": 1, "HitColor": "0", "LocalEventTime": "2013-07-11 05:19:46", "Age": 91, "Sex": 1, "Income": 3, "Interests": 227, "Robotness": 1, "RemoteIP": 2124359375, "WindowName": -421753, "OpenerName": 233007, "HistoryLength": 74, "BrowserLanguage": "\ufffd", "BrowserCountry": "\ufffd\f", "SocialNetwork": "", "SocialAction": "", "HTTPError": 1, "SendTiming": 2654, "DNSTiming": 90, "ConnectTiming": 767, "ResponseStartTiming": 1356, "ResponseEndTiming": 414, "FetchTiming": 745, "SocialSourceNetworkID": 2, "SocialSourcePage": "", "ParamPrice": "100.00", "ParamOrderID": "", "ParamCurrency": "EUR", "ParamCurrencyID": 5, "OpenstatServiceName": "", "OpenstatCampaignID": "", "OpenstatAdID": "", "OpenstatSourceID": "", "UTMSource": "google", "UTMMedium": "", "UTMCampaign": "summer_sale", "UTMContent": "", "UTMTerm": "", "FromTag": "", "HasGCLID": 1, "RefererHash": "-2963858722716533298", "URLHash": "1987944988283718819", "CLID": 15792} +{"index": {}} +{"WatchID": "1000000000000000004", "JavaEnable": 1, "Title": "Google Homepage", "GoodEvent": 0, "EventTime": "2013-07-02 16:45:00", "EventDate": "2013-07-01", "CounterID": 62, "ClientIP": 1234567893, "RegionID": 103, "UserID": "1000000000000000004", "CounterClass": 1, "OS": 4, "UserAgent": 53, "URL": "https://www.google.example.com/test", "Referer": "", "IsRefresh": 1, "RefererCategoryID": 13, "RefererRegionID": 103, "URLCategoryID": 23, "URLRegionID": 203, "ResolutionWidth": 1024, "ResolutionHeight": 768, "ResolutionDepth": 24, "FlashMajor": 11, "FlashMinor": 2, "FlashMinor2": "", "NetMajor": 4, "NetMinor": 1, "UserAgentMajor": 53, "UserAgentMinor": "0", "CookieEnable": 1, "JavascriptEnable": 0, "IsMobile": 0, "MobilePhone": 0, "MobilePhoneModel": "", "Params": "", "IPNetworkID": 1000003, "TraficSourceID": -1, "SearchEngineID": 3, "SearchPhrase": "test", "AdvEngineID": 2, "IsArtifical": 1, "WindowClientWidth": 1024, "WindowClientHeight": 768, "ClientTimeZone": 0, "ClientEventTime": "2013-07-02 16:45:00", "SilverlightVersion1": 0, "SilverlightVersion2": 0, "SilverlightVersion3": 0, "SilverlightVersion4": 0, "PageCharset": "utf-8", "CodeVersion": 3, "IsLink": 0, "IsDownload": 0, "IsNotBounce": 1, "FUniqID": "10000004", "OriginalURL": "", "HID": 100004, "IsOldCounter": 0, "IsEvent": 0, "IsParameter": 0, "DontCountHits": 0, "WithHash": 0, "HitColor": "5", "LocalEventTime": "2013-07-02 16:45:00", "Age": 40, "Sex": 2, "Income": 5, "Interests": 400, "Robotness": 0, "RemoteIP": 1234567893, "WindowName": 103, "OpenerName": 203, "HistoryLength": 20, "BrowserLanguage": "fr", "BrowserCountry": "FR", "SocialNetwork": "", "SocialAction": "", "HTTPError": 1, "SendTiming": 130, "DNSTiming": 65, "ConnectTiming": 115, "ResponseStartTiming": 215, "ResponseEndTiming": 315, "FetchTiming": 415, "SocialSourceNetworkID": 0, "SocialSourcePage": "", "ParamPrice": "25.50", "ParamOrderID": "", "ParamCurrency": "EUR", "ParamCurrencyID": 5, "OpenstatServiceName": "", "OpenstatCampaignID": "", "OpenstatAdID": "", "OpenstatSourceID": "", "UTMSource": "", "UTMMedium": "", "UTMCampaign": "", "UTMContent": "", "UTMTerm": "", "FromTag": "", "HasGCLID": 0, "RefererHash": "1000000000000000004", "URLHash": "2868770270353813622", "CLID": 1003} diff --git a/integ-test/src/test/resources/clickbench/mappings/clickbench_index_mapping.json b/integ-test/src/test/resources/clickbench/mappings/clickbench_index_mapping.json index 1482f8d8a5e..c695dcc0502 100644 --- a/integ-test/src/test/resources/clickbench/mappings/clickbench_index_mapping.json +++ b/integ-test/src/test/resources/clickbench/mappings/clickbench_index_mapping.json @@ -1,325 +1,333 @@ { "settings": { "index": { + "number_of_shards": 2, + "number_of_replicas": 0, "sort.field": [ "CounterID", "EventDate", "UserID", "EventTime", "WatchID" ], "sort.order": [ "desc", "desc", "desc", "desc", "desc" ] } }, "mappings" : { - "properties" : { - "AdvEngineID" : { - "type" : "short" - }, - "Age" : { - "type" : "short" - }, - "BrowserCountry" : { - "type" : "keyword" - }, - "BrowserLanguage" : { - "type" : "keyword" - }, - "CLID" : { - "type" : "integer" - }, - "ClientEventTime" : { - "type" : "date", - "format": "yyyy-MM-dd HH:mm:ss||strict_date_optional_time ||epoch_millis" - }, - "ClientIP" : { - "type" : "integer" - }, - "ClientTimeZone" : { - "type" : "short" - }, - "CodeVersion" : { - "type" : "integer" - }, - "ConnectTiming" : { - "type" : "integer" - }, - "CookieEnable" : { - "type" : "short" - }, - "CounterClass" : { - "type" : "short" - }, - "CounterID" : { - "type" : "integer" - }, - "DNSTiming" : { - "type" : "integer" - }, - "DontCountHits" : { - "type" : "short" - }, - "EventDate" : { - "type" : "date", - "format": "yyyy-MM-dd HH:mm:ss||strict_date_optional_time ||epoch_millis" - }, - "EventTime" : { - "type" : "date", - "format": "yyyy-MM-dd HH:mm:ss||strict_date_optional_time ||epoch_millis" - }, - "FUniqID" : { - "type" : "long" - }, - "FetchTiming" : { - "type" : "integer" - }, - "FlashMajor" : { - "type" : "short" - }, - "FlashMinor" : { - "type" : "short" - }, - "FlashMinor2" : { - "type" : "short" - }, - "FromTag" : { - "type" : "keyword" - }, - "GoodEvent" : { - "type" : "short" - }, - "HID" : { - "type" : "integer" - }, - "HTTPError" : { - "type" : "short" - }, - "HasGCLID" : { - "type" : "short" - }, - "HistoryLength" : { - "type" : "short" - }, - "HitColor" : { - "type" : "keyword" - }, - "IPNetworkID" : { - "type" : "integer" - }, - "Income" : { - "type" : "short" - }, - "Interests" : { - "type" : "short" - }, - "IsArtifical" : { - "type" : "short" - }, - "IsDownload" : { - "type" : "short" - }, - "IsEvent" : { - "type" : "short" - }, - "IsLink" : { - "type" : "short" - }, - "IsMobile" : { - "type" : "short" - }, - "IsNotBounce" : { - "type" : "short" - }, - "IsOldCounter" : { - "type" : "short" - }, - "IsParameter" : { - "type" : "short" - }, - "IsRefresh" : { - "type" : "short" - }, - "JavaEnable" : { - "type" : "short" - }, - "JavascriptEnable" : { - "type" : "short" - }, - "LocalEventTime" : { - "type" : "date", - "format": "yyyy-MM-dd HH:mm:ss||strict_date_optional_time ||epoch_millis" - }, - "MobilePhone" : { - "type" : "short" - }, - "MobilePhoneModel" : { - "type" : "keyword" - }, - "NetMajor" : { - "type" : "short" - }, - "NetMinor" : { - "type" : "short" - }, - "OS" : { - "type" : "short" - }, - "OpenerName" : { - "type" : "integer" - }, - "OpenstatAdID" : { - "type" : "keyword" - }, - "OpenstatCampaignID" : { - "type" : "keyword" - }, - "OpenstatServiceName" : { - "type" : "keyword" - }, - "OpenstatSourceID" : { - "type" : "keyword" - }, - "OriginalURL" : { - "type" : "keyword" - }, - "PageCharset" : { - "type" : "keyword" - }, - "ParamCurrency" : { - "type" : "keyword" - }, - "ParamCurrencyID" : { - "type" : "short" - }, - "ParamOrderID" : { - "type" : "keyword" - }, - "ParamPrice" : { - "type" : "long" - }, - "Params" : { - "type" : "keyword" - }, - "Referer" : { - "type" : "keyword" - }, - "RefererCategoryID" : { - "type" : "short" - }, - "RefererHash" : { - "type" : "long" - }, - "RefererRegionID" : { - "type" : "integer" - }, - "RegionID" : { - "type" : "integer" - }, - "RemoteIP" : { - "type" : "integer" - }, - "ResolutionDepth" : { - "type" : "short" - }, - "ResolutionHeight" : { - "type" : "short" - }, - "ResolutionWidth" : { - "type" : "short" - }, - "ResponseEndTiming" : { - "type" : "integer" - }, - "ResponseStartTiming" : { - "type" : "integer" - }, - "Robotness" : { - "type" : "short" - }, - "SearchEngineID" : { - "type" : "short" - }, - "SearchPhrase" : { - "type" : "keyword" - }, - "SendTiming" : { - "type" : "integer" - }, - "Sex" : { - "type" : "short" - }, - "SilverlightVersion1" : { - "type" : "short" - }, - "SilverlightVersion2" : { - "type" : "short" - }, - "SilverlightVersion3" : { - "type" : "integer" - }, - "SilverlightVersion4" : { - "type" : "short" - }, - "SocialSourceNetworkID" : { - "type" : "short" - }, - "SocialSourcePage" : { - "type" : "keyword" - }, - "Title" : { - "type" : "keyword" - }, - "TraficSourceID" : { - "type" : "short" - }, - "URL" : { - "type" : "keyword" - }, - "URLCategoryID" : { - "type" : "short" - }, - "URLHash" : { - "type" : "long" - }, - "URLRegionID" : { - "type" : "integer" - }, - "UTMCampaign" : { - "type" : "keyword" - }, - "UTMContent" : { - "type" : "keyword" - }, - "UTMMedium" : { - "type" : "keyword" - }, - "UTMSource" : { - "type" : "keyword" - }, - "UTMTerm" : { - "type" : "keyword" - }, - "UserAgent" : { - "type" : "short" - }, - "UserAgentMajor" : { - "type" : "short" - }, - "UserAgentMinor" : { - "type" : "keyword" - }, - "UserID" : { - "type" : "long" - }, - "WatchID" : { - "type" : "long" - }, - "WindowClientHeight" : { - "type" : "short" - }, - "WindowClientWidth" : { - "type" : "short" - }, - "WindowName" : { - "type" : "integer" - }, - "WithHash" : { - "type" : "short" - } - } - } + "properties": { + "AdvEngineID": { + "type": "short" + }, + "Age": { + "type": "short" + }, + "BrowserCountry": { + "type": "keyword" + }, + "SocialNetwork": { + "type": "keyword" + }, + "SocialAction": { + "type": "keyword" + }, + "BrowserLanguage": { + "type": "keyword" + }, + "CLID": { + "type": "integer" + }, + "ClientEventTime": { + "type": "date", + "format": "yyyy-MM-dd HH:mm:ss||strict_date_optional_time ||epoch_millis" + }, + "ClientIP": { + "type": "integer" + }, + "ClientTimeZone": { + "type": "short" + }, + "CodeVersion": { + "type": "integer" + }, + "ConnectTiming": { + "type": "integer" + }, + "CookieEnable": { + "type": "short" + }, + "CounterClass": { + "type": "short" + }, + "CounterID": { + "type": "integer" + }, + "DNSTiming": { + "type": "integer" + }, + "DontCountHits": { + "type": "short" + }, + "EventDate": { + "type": "date", + "format": "yyyy-MM-dd HH:mm:ss||strict_date_optional_time ||epoch_millis" + }, + "EventTime": { + "type": "date", + "format": "yyyy-MM-dd HH:mm:ss||strict_date_optional_time ||epoch_millis" + }, + "FUniqID": { + "type": "long" + }, + "FetchTiming": { + "type": "integer" + }, + "FlashMajor": { + "type": "short" + }, + "FlashMinor": { + "type": "short" + }, + "FlashMinor2": { + "type": "short" + }, + "FromTag": { + "type": "keyword" + }, + "GoodEvent": { + "type": "short" + }, + "HID": { + "type": "integer" + }, + "HTTPError": { + "type": "short" + }, + "HasGCLID": { + "type": "short" + }, + "HistoryLength": { + "type": "short" + }, + "HitColor": { + "type": "keyword" + }, + "IPNetworkID": { + "type": "integer" + }, + "Income": { + "type": "short" + }, + "Interests": { + "type": "short" + }, + "IsArtifical": { + "type": "short" + }, + "IsDownload": { + "type": "short" + }, + "IsEvent": { + "type": "short" + }, + "IsLink": { + "type": "short" + }, + "IsMobile": { + "type": "short" + }, + "IsNotBounce": { + "type": "short" + }, + "IsOldCounter": { + "type": "short" + }, + "IsParameter": { + "type": "short" + }, + "IsRefresh": { + "type": "short" + }, + "JavaEnable": { + "type": "short" + }, + "JavascriptEnable": { + "type": "short" + }, + "LocalEventTime": { + "type": "date", + "format": "yyyy-MM-dd HH:mm:ss||strict_date_optional_time ||epoch_millis" + }, + "MobilePhone": { + "type": "short" + }, + "MobilePhoneModel": { + "type": "keyword" + }, + "NetMajor": { + "type": "short" + }, + "NetMinor": { + "type": "short" + }, + "OS": { + "type": "short" + }, + "OpenerName": { + "type": "integer" + }, + "OpenstatAdID": { + "type": "keyword" + }, + "OpenstatCampaignID": { + "type": "keyword" + }, + "OpenstatServiceName": { + "type": "keyword" + }, + "OpenstatSourceID": { + "type": "keyword" + }, + "OriginalURL": { + "type": "keyword" + }, + "PageCharset": { + "type": "keyword" + }, + "ParamCurrency": { + "type": "keyword" + }, + "ParamCurrencyID": { + "type": "short" + }, + "ParamOrderID": { + "type": "keyword" + }, + "ParamPrice": { + "type": "long" + }, + "Params": { + "type": "keyword" + }, + "Referer": { + "type": "keyword" + }, + "RefererCategoryID": { + "type": "short" + }, + "RefererHash": { + "type": "long" + }, + "RefererRegionID": { + "type": "integer" + }, + "RegionID": { + "type": "integer" + }, + "RemoteIP": { + "type": "integer" + }, + "ResolutionDepth": { + "type": "short" + }, + "ResolutionHeight": { + "type": "short" + }, + "ResolutionWidth": { + "type": "short" + }, + "ResponseEndTiming": { + "type": "integer" + }, + "ResponseStartTiming": { + "type": "integer" + }, + "Robotness": { + "type": "short" + }, + "SearchEngineID": { + "type": "short" + }, + "SearchPhrase": { + "type": "keyword" + }, + "SendTiming": { + "type": "integer" + }, + "Sex": { + "type": "short" + }, + "SilverlightVersion1": { + "type": "short" + }, + "SilverlightVersion2": { + "type": "short" + }, + "SilverlightVersion3": { + "type": "integer" + }, + "SilverlightVersion4": { + "type": "short" + }, + "SocialSourceNetworkID": { + "type": "short" + }, + "SocialSourcePage": { + "type": "keyword" + }, + "Title": { + "type": "keyword" + }, + "TraficSourceID": { + "type": "short" + }, + "URL": { + "type": "keyword" + }, + "URLCategoryID": { + "type": "short" + }, + "URLHash": { + "type": "long" + }, + "URLRegionID": { + "type": "integer" + }, + "UTMCampaign": { + "type": "keyword" + }, + "UTMContent": { + "type": "keyword" + }, + "UTMMedium": { + "type": "keyword" + }, + "UTMSource": { + "type": "keyword" + }, + "UTMTerm": { + "type": "keyword" + }, + "UserAgent": { + "type": "short" + }, + "UserAgentMajor": { + "type": "short" + }, + "UserAgentMinor": { + "type": "keyword" + }, + "UserID": { + "type": "long" + }, + "WatchID": { + "type": "long" + }, + "WindowClientHeight": { + "type": "short" + }, + "WindowClientWidth": { + "type": "short" + }, + "WindowName": { + "type": "integer" + }, + "WithHash": { + "type": "short" + } + } + } } diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q1.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q1.json new file mode 100644 index 00000000000..158666d88f4 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q1.json @@ -0,0 +1,15 @@ +{ + "schema": [ + { + "name": "count()", + "type": "bigint" + } + ], + "datarows": [ + [ + 9 + ] + ], + "total": 1, + "size": 1 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q10.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q10.json new file mode 100644 index 00000000000..86b1441dddc --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q10.json @@ -0,0 +1,77 @@ +{ + "schema": [ + { + "name": "sum(AdvEngineID)", + "type": "bigint" + }, + { + "name": "c", + "type": "bigint" + }, + { + "name": "avg(ResolutionWidth)", + "type": "double" + }, + { + "name": "dc(UserID)", + "type": "bigint" + }, + { + "name": "RegionID", + "type": "int" + } + ], + "datarows": [ + [ + 4, + 2, + 1024.0, + 1, + 103 + ], + [ + 0, + 2, + 1920.0, + 2, + 104 + ], + [ + 0, + 1, + 1920.0, + 1, + 100 + ], + [ + 0, + 1, + 1366.0, + 1, + 101 + ], + [ + 1, + 1, + 1280.0, + 1, + 102 + ], + [ + 2, + 1, + 1920.0, + 1, + 331 + ], + [ + 4, + 1, + 1366.0, + 1, + 479 + ] + ], + "total": 7, + "size": 7 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q11.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q11.json new file mode 100644 index 00000000000..fa52c2cfe88 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q11.json @@ -0,0 +1,24 @@ +{ + "schema": [ + { + "name": "u", + "type": "bigint" + }, + { + "name": "MobilePhoneModel", + "type": "string" + } + ], + "datarows": [ + [ + 3, + "Samsung" + ], + [ + 1, + "iPhone" + ] + ], + "total": 2, + "size": 2 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q12.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q12.json new file mode 100644 index 00000000000..131541d2f8e --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q12.json @@ -0,0 +1,35 @@ +{ + "schema": [ + { + "name": "u", + "type": "bigint" + }, + { + "name": "MobilePhone", + "type": "smallint" + }, + { + "name": "MobilePhoneModel", + "type": "string" + } + ], + "datarows": [ + [ + 2, + 0, + "Samsung" + ], + [ + 1, + 1, + "Samsung" + ], + [ + 1, + 1, + "iPhone" + ] + ], + "total": 3, + "size": 3 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q13.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q13.json new file mode 100644 index 00000000000..30ae181e440 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q13.json @@ -0,0 +1,40 @@ +{ + "schema": [ + { + "name": "c", + "type": "bigint" + }, + { + "name": "SearchPhrase", + "type": "string" + } + ], + "datarows": [ + [ + 2, + "test" + ], + [ + 1, + "find something" + ], + [ + 1, + "google search" + ], + [ + 1, + "search something" + ], + [ + 1, + "test query" + ], + [ + 1, + "test query google" + ] + ], + "total": 6, + "size": 6 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q14.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q14.json new file mode 100644 index 00000000000..315d0220f0f --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q14.json @@ -0,0 +1,40 @@ +{ + "schema": [ + { + "name": "u", + "type": "bigint" + }, + { + "name": "SearchPhrase", + "type": "string" + } + ], + "datarows": [ + [ + 1, + "find something" + ], + [ + 1, + "google search" + ], + [ + 1, + "search something" + ], + [ + 1, + "test" + ], + [ + 1, + "test query" + ], + [ + 1, + "test query google" + ] + ], + "total": 6, + "size": 6 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q15.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q15.json new file mode 100644 index 00000000000..3a3bc23987b --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q15.json @@ -0,0 +1,50 @@ +{ + "schema": [ + { + "name": "c", + "type": "bigint" + }, + { + "name": "SearchEngineID", + "type": "smallint" + }, + { + "name": "SearchPhrase", + "type": "string" + } + ], + "datarows": [ + [ + 2, + 3, + "test" + ], + [ + 1, + 1, + "find something" + ], + [ + 1, + 1, + "google search" + ], + [ + 1, + 1, + "test query" + ], + [ + 1, + 1, + "test query google" + ], + [ + 1, + 2, + "search something" + ] + ], + "total": 6, + "size": 6 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q16.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q16.json new file mode 100644 index 00000000000..25f8094b750 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q16.json @@ -0,0 +1,48 @@ +{ + "schema": [ + { + "name": "count()", + "type": "bigint" + }, + { + "name": "UserID", + "type": "bigint" + } + ], + "datarows": [ + [ + 2, + 1000000000000000004 + ], + [ + 1, + -6563107575738400561 + ], + [ + 1, + 435090932899640449 + ], + [ + 1, + 1000000000000000001 + ], + [ + 1, + 1000000000000000002 + ], + [ + 1, + 1000000000000000003 + ], + [ + 1, + 1000000000000000005 + ], + [ + 1, + 6515901508915708000 + ] + ], + "total": 8, + "size": 8 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q17.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q17.json new file mode 100644 index 00000000000..b9626758ff7 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q17.json @@ -0,0 +1,60 @@ +{ + "schema": [ + { + "name": "count()", + "type": "bigint" + }, + { + "name": "UserID", + "type": "bigint" + }, + { + "name": "SearchPhrase", + "type": "string" + } + ], + "datarows": [ + [ + 2, + 1000000000000000004, + "test" + ], + [ + 1, + -6563107575738400561, + "test query" + ], + [ + 1, + 435090932899640449, + "" + ], + [ + 1, + 1000000000000000001, + "test query google" + ], + [ + 1, + 1000000000000000002, + "search something" + ], + [ + 1, + 1000000000000000003, + "google search" + ], + [ + 1, + 1000000000000000005, + "find something" + ], + [ + 1, + 6515901508915708000, + "" + ] + ], + "total": 8, + "size": 8 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q18.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q18.json new file mode 100644 index 00000000000..cf92162df92 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q18.json @@ -0,0 +1,60 @@ +{ + "schema": [ + { + "name": "count()", + "type": "bigint" + }, + { + "name": "UserID", + "type": "bigint" + }, + { + "name": "SearchPhrase", + "type": "string" + } + ], + "datarows": [ + [ + 1, + 435090932899640449, + "" + ], + [ + 1, + 6515901508915708000, + "" + ], + [ + 1, + 1000000000000000005, + "find something" + ], + [ + 1, + 1000000000000000003, + "google search" + ], + [ + 1, + 1000000000000000002, + "search something" + ], + [ + 2, + 1000000000000000004, + "test" + ], + [ + 1, + -6563107575738400561, + "test query" + ], + [ + 1, + 1000000000000000001, + "test query google" + ] + ], + "total": 8, + "size": 8 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q19.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q19.json new file mode 100644 index 00000000000..42da3d8a8c0 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q19.json @@ -0,0 +1,72 @@ +{ + "schema": [ + { + "name": "count()", + "type": "bigint" + }, + { + "name": "UserID", + "type": "bigint" + }, + { + "name": "m", + "type": "bigint" + }, + { + "name": "SearchPhrase", + "type": "string" + } + ], + "datarows": [ + [ + 2, + 1000000000000000004, + 45, + "test" + ], + [ + 1, + 435090932899640449, + 58, + "" + ], + [ + 1, + 6515901508915708000, + 50, + "" + ], + [ + 1, + 1000000000000000005, + 30, + "find something" + ], + [ + 1, + 1000000000000000003, + 15, + "google search" + ], + [ + 1, + 1000000000000000002, + 20, + "search something" + ], + [ + 1, + -6563107575738400561, + 19, + "test query" + ], + [ + 1, + 1000000000000000001, + 30, + "test query google" + ] + ], + "total": 8, + "size": 8 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q2.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q2.json new file mode 100644 index 00000000000..c2aea6df188 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q2.json @@ -0,0 +1,15 @@ +{ + "schema": [ + { + "name": "count()", + "type": "bigint" + } + ], + "datarows": [ + [ + 5 + ] + ], + "total": 1, + "size": 1 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q20.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q20.json new file mode 100644 index 00000000000..5807f090f99 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q20.json @@ -0,0 +1,15 @@ +{ + "schema": [ + { + "name": "UserID", + "type": "bigint" + } + ], + "datarows": [ + [ + 435090932899640449 + ] + ], + "total": 1, + "size": 1 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q21.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q21.json new file mode 100644 index 00000000000..17a684d5bbe --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q21.json @@ -0,0 +1,15 @@ +{ + "schema": [ + { + "name": "count()", + "type": "bigint" + } + ], + "datarows": [ + [ + 6 + ] + ], + "total": 1, + "size": 1 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q22.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q22.json new file mode 100644 index 00000000000..b6dd5a17680 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q22.json @@ -0,0 +1,36 @@ +{ + "schema": [ + { + "name": "c", + "type": "bigint" + }, + { + "name": "SearchPhrase", + "type": "string" + } + ], + "datarows": [ + [ + 2, + "test" + ], + [ + 1, + "find something" + ], + [ + 1, + "google search" + ], + [ + 1, + "search something" + ], + [ + 1, + "test query google" + ] + ], + "total": 5, + "size": 5 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q23.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q23.json new file mode 100644 index 00000000000..107b392e299 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q23.json @@ -0,0 +1,30 @@ +{ + "schema": [ + { + "name": "c", + "type": "bigint" + }, + { + "name": "dc(UserID)", + "type": "bigint" + }, + { + "name": "SearchPhrase", + "type": "string" + } + ], + "datarows": [ + [ + 1, + 1, + "google search" + ], + [ + 1, + 1, + "search something" + ] + ], + "total": 2, + "size": 2 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q24.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q24.json new file mode 100644 index 00000000000..0365a4d63d0 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q24.json @@ -0,0 +1,1070 @@ +{ + "schema": [ + { + "name": "EventDate", + "type": "timestamp" + }, + { + "name": "URLRegionID", + "type": "int" + }, + { + "name": "HasGCLID", + "type": "smallint" + }, + { + "name": "Income", + "type": "smallint" + }, + { + "name": "Interests", + "type": "smallint" + }, + { + "name": "Robotness", + "type": "smallint" + }, + { + "name": "BrowserLanguage", + "type": "string" + }, + { + "name": "CounterClass", + "type": "smallint" + }, + { + "name": "BrowserCountry", + "type": "string" + }, + { + "name": "OriginalURL", + "type": "string" + }, + { + "name": "ClientTimeZone", + "type": "smallint" + }, + { + "name": "RefererHash", + "type": "bigint" + }, + { + "name": "TraficSourceID", + "type": "smallint" + }, + { + "name": "HitColor", + "type": "string" + }, + { + "name": "RefererRegionID", + "type": "int" + }, + { + "name": "URLCategoryID", + "type": "smallint" + }, + { + "name": "LocalEventTime", + "type": "timestamp" + }, + { + "name": "EventTime", + "type": "timestamp" + }, + { + "name": "UTMTerm", + "type": "string" + }, + { + "name": "AdvEngineID", + "type": "smallint" + }, + { + "name": "UserAgentMinor", + "type": "string" + }, + { + "name": "UserAgentMajor", + "type": "smallint" + }, + { + "name": "RemoteIP", + "type": "int" + }, + { + "name": "Sex", + "type": "smallint" + }, + { + "name": "JavaEnable", + "type": "smallint" + }, + { + "name": "URLHash", + "type": "bigint" + }, + { + "name": "URL", + "type": "string" + }, + { + "name": "ParamOrderID", + "type": "string" + }, + { + "name": "OpenstatSourceID", + "type": "string" + }, + { + "name": "HTTPError", + "type": "smallint" + }, + { + "name": "SilverlightVersion3", + "type": "int" + }, + { + "name": "MobilePhoneModel", + "type": "string" + }, + { + "name": "SilverlightVersion4", + "type": "smallint" + }, + { + "name": "SilverlightVersion1", + "type": "smallint" + }, + { + "name": "SilverlightVersion2", + "type": "smallint" + }, + { + "name": "IsDownload", + "type": "smallint" + }, + { + "name": "IsParameter", + "type": "smallint" + }, + { + "name": "CLID", + "type": "int" + }, + { + "name": "FlashMajor", + "type": "smallint" + }, + { + "name": "FlashMinor", + "type": "smallint" + }, + { + "name": "UTMMedium", + "type": "string" + }, + { + "name": "WatchID", + "type": "bigint" + }, + { + "name": "DontCountHits", + "type": "smallint" + }, + { + "name": "CookieEnable", + "type": "smallint" + }, + { + "name": "HID", + "type": "int" + }, + { + "name": "SocialAction", + "type": "string" + }, + { + "name": "WindowName", + "type": "int" + }, + { + "name": "ConnectTiming", + "type": "int" + }, + { + "name": "PageCharset", + "type": "string" + }, + { + "name": "IsLink", + "type": "smallint" + }, + { + "name": "IsArtifical", + "type": "smallint" + }, + { + "name": "JavascriptEnable", + "type": "smallint" + }, + { + "name": "ClientEventTime", + "type": "timestamp" + }, + { + "name": "DNSTiming", + "type": "int" + }, + { + "name": "CodeVersion", + "type": "int" + }, + { + "name": "ResponseEndTiming", + "type": "int" + }, + { + "name": "FUniqID", + "type": "bigint" + }, + { + "name": "WindowClientHeight", + "type": "smallint" + }, + { + "name": "OpenstatServiceName", + "type": "string" + }, + { + "name": "UTMContent", + "type": "string" + }, + { + "name": "HistoryLength", + "type": "smallint" + }, + { + "name": "IsOldCounter", + "type": "smallint" + }, + { + "name": "MobilePhone", + "type": "smallint" + }, + { + "name": "SearchPhrase", + "type": "string" + }, + { + "name": "FlashMinor2", + "type": "smallint" + }, + { + "name": "SearchEngineID", + "type": "smallint" + }, + { + "name": "IsEvent", + "type": "smallint" + }, + { + "name": "UTMSource", + "type": "string" + }, + { + "name": "RegionID", + "type": "int" + }, + { + "name": "OpenstatAdID", + "type": "string" + }, + { + "name": "UTMCampaign", + "type": "string" + }, + { + "name": "GoodEvent", + "type": "smallint" + }, + { + "name": "IsRefresh", + "type": "smallint" + }, + { + "name": "ParamCurrency", + "type": "string" + }, + { + "name": "Params", + "type": "string" + }, + { + "name": "ResolutionHeight", + "type": "smallint" + }, + { + "name": "ClientIP", + "type": "int" + }, + { + "name": "FromTag", + "type": "string" + }, + { + "name": "ParamCurrencyID", + "type": "smallint" + }, + { + "name": "ResponseStartTiming", + "type": "int" + }, + { + "name": "ResolutionWidth", + "type": "smallint" + }, + { + "name": "SendTiming", + "type": "int" + }, + { + "name": "RefererCategoryID", + "type": "smallint" + }, + { + "name": "OpenstatCampaignID", + "type": "string" + }, + { + "name": "UserID", + "type": "bigint" + }, + { + "name": "WithHash", + "type": "smallint" + }, + { + "name": "UserAgent", + "type": "smallint" + }, + { + "name": "ParamPrice", + "type": "bigint" + }, + { + "name": "ResolutionDepth", + "type": "smallint" + }, + { + "name": "IsMobile", + "type": "smallint" + }, + { + "name": "Age", + "type": "smallint" + }, + { + "name": "SocialSourceNetworkID", + "type": "smallint" + }, + { + "name": "OpenerName", + "type": "int" + }, + { + "name": "OS", + "type": "smallint" + }, + { + "name": "IsNotBounce", + "type": "smallint" + }, + { + "name": "Referer", + "type": "string" + }, + { + "name": "NetMinor", + "type": "smallint" + }, + { + "name": "Title", + "type": "string" + }, + { + "name": "NetMajor", + "type": "smallint" + }, + { + "name": "IPNetworkID", + "type": "int" + }, + { + "name": "FetchTiming", + "type": "int" + }, + { + "name": "SocialNetwork", + "type": "string" + }, + { + "name": "SocialSourcePage", + "type": "string" + }, + { + "name": "CounterID", + "type": "int" + }, + { + "name": "WindowClientWidth", + "type": "smallint" + } + ], + "datarows": [ + [ + "2013-07-02 00:00:00", + 203, + 0, + 5, + 400, + 0, + "fr", + 1, + "FR", + "", + 0, + 1000000000000000004, + -1, + "5", + 103, + 23, + "2013-07-02 16:45:00", + "2013-07-02 16:45:00", + "", + 2, + "0", + 53, + 1234567893, + 2, + 1, + 2868770270353813622, + "https://www.google.example.com/test", + "", + "", + 1, + 0, + "", + 0, + 0, + 0, + 0, + 0, + 1003, + 11, + 2, + "", + 1000000000000000004, + 0, + 1, + 100004, + "", + 103, + 115, + "utf-8", + 0, + 1, + 0, + "2013-07-02 16:45:00", + 65, + 3, + 315, + 10000004, + 768, + "", + "", + 20, + 0, + 0, + "test", + 0, + 3, + 0, + "", + 103, + "", + "", + 0, + 0, + "EUR", + "", + 768, + 1234567893, + "", + 5, + 215, + 1024, + 130, + 13, + "", + 1000000000000000004, + 0, + 53, + 25, + 24, + 0, + 40, + 0, + 203, + 4, + 1, + "", + 1, + "Google Homepage", + 4, + 1000003, + 415, + "", + "", + 62, + 1024 + ], + [ + "2013-07-01 00:00:00", + 203, + 0, + 5, + 400, + 0, + "fr", + 1, + "FR", + "", + 0, + 1000000000000000004, + -1, + "5", + 103, + 23, + "2013-07-02 16:45:00", + "2013-07-02 16:45:00", + "", + 2, + "0", + 53, + 1234567893, + 2, + 1, + 2868770270353813622, + "https://www.google.example.com/test", + "", + "", + 1, + 0, + "", + 0, + 0, + 0, + 0, + 0, + 1003, + 11, + 2, + "", + 1000000000000000004, + 0, + 1, + 100004, + "", + 103, + 115, + "utf-8", + 0, + 1, + 0, + "2013-07-02 16:45:00", + 65, + 3, + 315, + 10000004, + 768, + "", + "", + 20, + 0, + 0, + "test", + 0, + 3, + 0, + "", + 103, + "", + "", + 0, + 1, + "EUR", + "", + 768, + 1234567893, + "", + 5, + 215, + 1024, + 130, + 13, + "", + 1000000000000000004, + 0, + 53, + 25, + 24, + 0, + 40, + 0, + 203, + 4, + 1, + "", + 1, + "Google Homepage", + 4, + 1000003, + 415, + "", + "", + 62, + 1024 + ], + [ + "2013-07-05 00:00:00", + 204, + 0, + 3, + 500, + 1, + "de", + 0, + "DE", + "", + 8, + 3594120000172545465, + 6, + "1", + 104, + 24, + "2013-07-05 11:30:00", + "2013-07-05 11:30:00", + "", + 0, + "0", + 54, + 1234567894, + 1, + 1, + 1000000000000000005, + "https://example.google.org/page", + "ORDER002", + "", + 0, + 0, + "Samsung", + 0, + 0, + 0, + 1, + 1, + 1004, + 10, + 1, + "", + 1000000000000000005, + 0, + 1, + 100005, + "", + 104, + 120, + "utf-8", + 1, + 0, + 1, + "2013-07-05 11:30:00", + 70, + 4, + 320, + 10000005, + 1200, + "", + "", + 8, + 1, + 0, + "find something", + 0, + 1, + 1, + "", + 104, + "", + "", + 1, + 0, + "EUR", + "", + 1200, + 1234567894, + "", + 5, + 220, + 1920, + 140, + 14, + "", + 1000000000000000005, + 1, + 54, + 100, + 32, + 1, + 28, + 0, + 204, + 5, + 1, + "https://google.com", + 7, + "Page with Google Reference", + 3, + 1000004, + 420, + "", + "", + 62, + 1920 + ], + [ + "2013-07-10 00:00:00", + 202, + 1, + 4, + 300, + 1, + "en", + 0, + "US", + "", + 5, + 1000000000000000003, + 6, + "0", + 102, + 22, + "2013-07-10 09:15:00", + "2013-07-10 09:15:00", + "", + 1, + "0", + 52, + 1234567892, + 1, + 0, + 2868770270353813622, + "https://example.com/page?ref=google", + "ORDER001", + "", + 0, + 0, + "iPhone", + 0, + 0, + 0, + 0, + 1, + 1002, + 9, + 3, + "organic", + 1000000000000000003, + 0, + 0, + 100003, + "", + 102, + 110, + "utf-8", + 1, + 0, + 1, + "2013-07-10 09:15:00", + 60, + 2, + 310, + 10000003, + 1024, + "", + "", + 15, + 1, + 1, + "google search", + 0, + 1, + 1, + "google", + 102, + "", + "", + 1, + 0, + "USD", + "", + 1024, + 1234567892, + "", + 1, + 210, + 1280, + 120, + 12, + "", + 1000000000000000003, + 1, + 52, + 10, + 32, + 1, + 35, + 0, + 202, + 3, + 0, + "https://www.google.com", + 8, + "Welcome to Google", + 2, + 1000002, + 410, + "", + "", + 62, + 1280 + ], + [ + "2013-07-14 00:00:00", + 201, + 0, + 2, + 200, + 0, + "en", + 1, + "UK", + "", + -5, + 3594120000172545465, + -1, + "2", + 101, + 21, + "2013-07-14 14:20:00", + "2013-07-14 14:20:00", + "", + 0, + "0", + 51, + 1234567891, + 2, + 1, + 1000000000000000002, + "https://google.com/maps", + "", + "", + 0, + 0, + "", + 0, + 0, + 0, + 0, + 0, + 1001, + 10, + 4, + "", + 1000000000000000002, + 0, + 1, + 100002, + "", + 101, + 105, + "utf-8", + 0, + 0, + 1, + "2013-07-14 14:20:00", + 55, + 1, + 305, + 10000002, + 768, + "", + "", + 5, + 0, + 0, + "search something", + 0, + 2, + 0, + "", + 101, + "", + "", + 1, + 0, + "GBP", + "", + 768, + 1234567891, + "", + 3, + 205, + 1366, + 110, + 11, + "", + 1000000000000000002, + 0, + 51, + 0, + 24, + 0, + 25, + 0, + 201, + 2, + 1, + "https://example.com", + 9, + "Google Search Page", + 3, + 1000001, + 405, + "", + "", + 62, + 1366 + ], + [ + "2013-07-15 00:00:00", + 200, + 0, + 3, + 100, + 0, + "en", + 1, + "US", + "", + 0, + 3594120000172545465, + 6, + "1", + 100, + 20, + "2013-07-15 10:30:00", + "2013-07-15 10:30:00", + "", + 0, + "0", + 50, + 1234567890, + 1, + 1, + 1000000000000000001, + "https://www.google.com/search?q=test", + "", + "", + 0, + 0, + "", + 0, + 0, + 0, + 0, + 0, + 1000, + 11, + 5, + "", + 1000000000000000001, + 0, + 1, + 100001, + "", + 100, + 100, + "utf-8", + 1, + 0, + 1, + "2013-07-15 10:30:00", + 50, + 1, + 300, + 10000001, + 1080, + "", + "", + 10, + 0, + 0, + "test query google", + 0, + 1, + 1, + "", + 100, + "", + "", + 1, + 0, + "USD", + "", + 1080, + 1234567890, + "", + 1, + 200, + 1920, + 100, + 10, + "", + 1000000000000000001, + 0, + 50, + 0, + 24, + 0, + 30, + 0, + 200, + 1, + 1, + "https://example.com", + 0, + "Search Results", + 4, + 1000000, + 400, + "", + "", + 62, + 1920 + ] + ], + "total": 6, + "size": 6 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q25.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q25.json new file mode 100644 index 00000000000..84beaa8b213 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q25.json @@ -0,0 +1,33 @@ +{ + "schema": [ + { + "name": "SearchPhrase", + "type": "string" + } + ], + "datarows": [ + [ + "test" + ], + [ + "test" + ], + [ + "find something" + ], + [ + "google search" + ], + [ + "test query" + ], + [ + "search something" + ], + [ + "test query google" + ] + ], + "total": 7, + "size": 7 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q26.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q26.json new file mode 100644 index 00000000000..d62fe2056a4 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q26.json @@ -0,0 +1,33 @@ +{ + "schema": [ + { + "name": "SearchPhrase", + "type": "string" + } + ], + "datarows": [ + [ + "find something" + ], + [ + "google search" + ], + [ + "search something" + ], + [ + "test" + ], + [ + "test" + ], + [ + "test query" + ], + [ + "test query google" + ] + ], + "total": 7, + "size": 7 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q27.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q27.json new file mode 100644 index 00000000000..84beaa8b213 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q27.json @@ -0,0 +1,33 @@ +{ + "schema": [ + { + "name": "SearchPhrase", + "type": "string" + } + ], + "datarows": [ + [ + "test" + ], + [ + "test" + ], + [ + "find something" + ], + [ + "google search" + ], + [ + "test query" + ], + [ + "search something" + ], + [ + "test query google" + ] + ], + "total": 7, + "size": 7 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q28.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q28.json new file mode 100644 index 00000000000..ee989fad51c --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q28.json @@ -0,0 +1,25 @@ +{ + "schema": [ + { + "name": "l", + "type": "double" + }, + { + "name": "c", + "type": "bigint" + }, + { + "name": "CounterID", + "type": "int" + } + ], + "datarows": [ + [ + 32.5, + 6, + 62 + ] + ], + "total": 1, + "size": 1 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q29.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q29.json new file mode 100644 index 00000000000..d101d8c1a14 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q29.json @@ -0,0 +1,30 @@ +{ + "schema": [ + { + "name": "l", + "type": "double" + }, + { + "name": "c", + "type": "bigint" + }, + { + "name": "min(Referer)", + "type": "string" + }, + { + "name": "k", + "type": "string" + } + ], + "datarows": [ + [ + 19.0, + 2, + "https://example.com", + "https://example.com" + ] + ], + "total": 1, + "size": 1 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q3.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q3.json new file mode 100644 index 00000000000..1a58447651b --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q3.json @@ -0,0 +1,25 @@ +{ + "schema": [ + { + "name": "sum(AdvEngineID)", + "type": "bigint" + }, + { + "name": "count()", + "type": "bigint" + }, + { + "name": "avg(ResolutionWidth)", + "type": "double" + } + ], + "datarows": [ + [ + 11, + 9, + 1526.6666666666667 + ] + ], + "total": 1, + "size": 1 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q30.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q30.json new file mode 100644 index 00000000000..a2d25ce49a5 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q30.json @@ -0,0 +1,460 @@ +{ + "schema": [ + { + "name": "sum(ResolutionWidth)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+1)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+2)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+3)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+4)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+5)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+6)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+7)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+8)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+9)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+10)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+11)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+12)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+13)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+14)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+15)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+16)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+17)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+18)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+19)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+20)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+21)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+22)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+23)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+24)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+25)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+26)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+27)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+28)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+29)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+30)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+31)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+32)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+33)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+34)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+35)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+36)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+37)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+38)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+39)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+40)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+41)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+42)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+43)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+44)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+45)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+46)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+47)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+48)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+49)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+50)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+51)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+52)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+53)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+54)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+55)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+56)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+57)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+58)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+59)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+60)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+61)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+62)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+63)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+64)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+65)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+66)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+67)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+68)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+69)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+70)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+71)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+72)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+73)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+74)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+75)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+76)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+77)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+78)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+79)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+80)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+81)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+82)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+83)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+84)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+85)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+86)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+87)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+88)", + "type": "bigint" + }, + { + "name": "sum(ResolutionWidth+89)", + "type": "bigint" + } + ], + "datarows": [ + [ + 13740, + 13749, + 13758, + 13767, + 13776, + 13785, + 13794, + 13803, + 13812, + 13821, + 13830, + 13839, + 13848, + 13857, + 13866, + 13875, + 13884, + 13893, + 13902, + 13911, + 13920, + 13929, + 13938, + 13947, + 13956, + 13965, + 13974, + 13983, + 13992, + 14001, + 14010, + 14019, + 14028, + 14037, + 14046, + 14055, + 14064, + 14073, + 14082, + 14091, + 14100, + 14109, + 14118, + 14127, + 14136, + 14145, + 14154, + 14163, + 14172, + 14181, + 14190, + 14199, + 14208, + 14217, + 14226, + 14235, + 14244, + 14253, + 14262, + 14271, + 14280, + 14289, + 14298, + 14307, + 14316, + 14325, + 14334, + 14343, + 14352, + 14361, + 14370, + 14379, + 14388, + 14397, + 14406, + 14415, + 14424, + 14433, + 14442, + 14451, + 14460, + 14469, + 14478, + 14487, + 14496, + 14505, + 14514, + 14523, + 14532, + 14541 + ] + ], + "total": 1, + "size": 1 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q31.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q31.json new file mode 100644 index 00000000000..bbd4a40680f --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q31.json @@ -0,0 +1,70 @@ +{ + "schema": [ + { + "name": "c", + "type": "bigint" + }, + { + "name": "sum(IsRefresh)", + "type": "bigint" + }, + { + "name": "avg(ResolutionWidth)", + "type": "double" + }, + { + "name": "SearchEngineID", + "type": "smallint" + }, + { + "name": "ClientIP", + "type": "int" + } + ], + "datarows": [ + [ + 2, + 1, + 1024.0, + 3, + 1234567893 + ], + [ + 1, + 1, + 1366.0, + 1, + 437216268 + ], + [ + 1, + 0, + 1920.0, + 1, + 1234567890 + ], + [ + 1, + 0, + 1280.0, + 1, + 1234567892 + ], + [ + 1, + 0, + 1920.0, + 1, + 1234567894 + ], + [ + 1, + 0, + 1366.0, + 2, + 1234567891 + ] + ], + "total": 6, + "size": 6 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q32.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q32.json new file mode 100644 index 00000000000..7ae577b23a9 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q32.json @@ -0,0 +1,70 @@ +{ + "schema": [ + { + "name": "c", + "type": "bigint" + }, + { + "name": "sum(IsRefresh)", + "type": "bigint" + }, + { + "name": "avg(ResolutionWidth)", + "type": "double" + }, + { + "name": "WatchID", + "type": "bigint" + }, + { + "name": "ClientIP", + "type": "int" + } + ], + "datarows": [ + [ + 2, + 1, + 1024.0, + 1000000000000000004, + 1234567893 + ], + [ + 1, + 0, + 1920.0, + 1000000000000000001, + 1234567890 + ], + [ + 1, + 0, + 1366.0, + 1000000000000000002, + 1234567891 + ], + [ + 1, + 0, + 1280.0, + 1000000000000000003, + 1234567892 + ], + [ + 1, + 0, + 1920.0, + 1000000000000000005, + 1234567894 + ], + [ + 1, + 1, + 1366.0, + 2133884852119575301, + 437216268 + ] + ], + "total": 6, + "size": 6 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q33.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q33.json new file mode 100644 index 00000000000..31fe01dd210 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q33.json @@ -0,0 +1,84 @@ +{ + "schema": [ + { + "name": "c", + "type": "bigint" + }, + { + "name": "sum(IsRefresh)", + "type": "bigint" + }, + { + "name": "avg(ResolutionWidth)", + "type": "double" + }, + { + "name": "WatchID", + "type": "bigint" + }, + { + "name": "ClientIP", + "type": "int" + } + ], + "datarows": [ + [ + 2, + 1, + 1024.0, + 1000000000000000004, + 1234567893 + ], + [ + 1, + 0, + 1920.0, + 1000000000000000001, + 1234567890 + ], + [ + 1, + 0, + 1366.0, + 1000000000000000002, + 1234567891 + ], + [ + 1, + 0, + 1280.0, + 1000000000000000003, + 1234567892 + ], + [ + 1, + 0, + 1920.0, + 1000000000000000005, + 1234567894 + ], + [ + 1, + 1, + 1366.0, + 2133884852119575301, + 437216268 + ], + [ + 1, + 1, + 1920.0, + 2203335828757764551, + 2092362178 + ], + [ + 1, + 0, + 1920.0, + 2310139442600589659, + -902092651 + ] + ], + "total": 8, + "size": 8 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q34.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q34.json new file mode 100644 index 00000000000..0e2d1e1897a --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q34.json @@ -0,0 +1,48 @@ +{ + "schema": [ + { + "name": "c", + "type": "bigint" + }, + { + "name": "URL", + "type": "string" + } + ], + "datarows": [ + [ + 2, + "https://www.google.example.com/test" + ], + [ + 1, + "https://example.com/about" + ], + [ + 1, + "https://example.com/page?ref=google" + ], + [ + 1, + "https://example.com/products" + ], + [ + 1, + "https://example.com/search" + ], + [ + 1, + "https://example.google.org/page" + ], + [ + 1, + "https://google.com/maps" + ], + [ + 1, + "https://www.google.com/search?q=test" + ] + ], + "total": 8, + "size": 8 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q35.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q35.json new file mode 100644 index 00000000000..0afe59c6080 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q35.json @@ -0,0 +1,60 @@ +{ + "schema": [ + { + "name": "c", + "type": "bigint" + }, + { + "name": "const", + "type": "int" + }, + { + "name": "URL", + "type": "string" + } + ], + "datarows": [ + [ + 2, + 1, + "https://www.google.example.com/test" + ], + [ + 1, + 1, + "https://example.com/about" + ], + [ + 1, + 1, + "https://example.com/page?ref=google" + ], + [ + 1, + 1, + "https://example.com/products" + ], + [ + 1, + 1, + "https://example.com/search" + ], + [ + 1, + 1, + "https://example.google.org/page" + ], + [ + 1, + 1, + "https://google.com/maps" + ], + [ + 1, + 1, + "https://www.google.com/search?q=test" + ] + ], + "total": 8, + "size": 8 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q36.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q36.json new file mode 100644 index 00000000000..52c17860d3f --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q36.json @@ -0,0 +1,84 @@ +{ + "schema": [ + { + "name": "c", + "type": "bigint" + }, + { + "name": "ClientIP", + "type": "int" + }, + { + "name": "ClientIP - 1", + "type": "int" + }, + { + "name": "ClientIP - 2", + "type": "int" + }, + { + "name": "ClientIP - 3", + "type": "int" + } + ], + "datarows": [ + [ + 2, + 1234567893, + 1234567892, + 1234567891, + 1234567890 + ], + [ + 1, + -902092651, + -902092652, + -902092653, + -902092654 + ], + [ + 1, + 437216268, + 437216267, + 437216266, + 437216265 + ], + [ + 1, + 1234567890, + 1234567889, + 1234567888, + 1234567887 + ], + [ + 1, + 1234567891, + 1234567890, + 1234567889, + 1234567888 + ], + [ + 1, + 1234567892, + 1234567891, + 1234567890, + 1234567889 + ], + [ + 1, + 1234567894, + 1234567893, + 1234567892, + 1234567891 + ], + [ + 1, + 2092362178, + 2092362177, + 2092362176, + 2092362175 + ] + ], + "total": 8, + "size": 8 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q37.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q37.json new file mode 100644 index 00000000000..60f1c27676a --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q37.json @@ -0,0 +1,36 @@ +{ + "schema": [ + { + "name": "PageViews", + "type": "bigint" + }, + { + "name": "URL", + "type": "string" + } + ], + "datarows": [ + [ + 1, + "https://example.com/page?ref=google" + ], + [ + 1, + "https://example.google.org/page" + ], + [ + 1, + "https://google.com/maps" + ], + [ + 1, + "https://www.google.com/search?q=test" + ], + [ + 1, + "https://www.google.example.com/test" + ] + ], + "total": 5, + "size": 5 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q38.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q38.json new file mode 100644 index 00000000000..59c5f8c633e --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q38.json @@ -0,0 +1,36 @@ +{ + "schema": [ + { + "name": "PageViews", + "type": "bigint" + }, + { + "name": "Title", + "type": "string" + } + ], + "datarows": [ + [ + 1, + "Google Homepage" + ], + [ + 1, + "Google Search Page" + ], + [ + 1, + "Page with Google Reference" + ], + [ + 1, + "Search Results" + ], + [ + 1, + "Welcome to Google" + ] + ], + "total": 5, + "size": 5 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q39.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q39.json new file mode 100644 index 00000000000..c3893e79a1a --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q39.json @@ -0,0 +1,20 @@ +{ + "schema": [ + { + "name": "PageViews", + "type": "bigint" + }, + { + "name": "URL", + "type": "string" + } + ], + "datarows": [ + [ + 1, + "https://www.google.com/search?q=test" + ] + ], + "total": 1, + "size": 1 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q4.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q4.json new file mode 100644 index 00000000000..027876bb7fa --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q4.json @@ -0,0 +1,15 @@ +{ + "schema": [ + { + "name": "avg(UserID)", + "type": "double" + } + ], + "datarows": [ + [ + 7.097649851196608E17 + ] + ], + "total": 1, + "size": 1 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q40.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q40.json new file mode 100644 index 00000000000..3da5b5d205b --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q40.json @@ -0,0 +1,64 @@ +{ + "schema": [ + { + "name": "PageViews", + "type": "bigint" + }, + { + "name": "TraficSourceID", + "type": "smallint" + }, + { + "name": "SearchEngineID", + "type": "smallint" + }, + { + "name": "AdvEngineID", + "type": "smallint" + }, + { + "name": "Src", + "type": "string" + }, + { + "name": "Dst", + "type": "string" + } + ], + "datarows": [ + [ + 1, + -1, + 3, + 2, + "", + "https://www.google.example.com/test" + ], + [ + 1, + 6, + 1, + 0, + "", + "https://example.google.org/page" + ], + [ + 1, + 6, + 1, + 0, + "", + "https://www.google.com/search?q=test" + ], + [ + 1, + 6, + 1, + 1, + "", + "https://example.com/page?ref=google" + ] + ], + "total": 4, + "size": 4 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q41.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q41.json new file mode 100644 index 00000000000..7da30317771 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q41.json @@ -0,0 +1 @@ +THIS QUERY TEST CURRENTLY FAILS ON MAIN WHEN THERE IS A HIT diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q42.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q42.json new file mode 100644 index 00000000000..e4c852b2c16 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q42.json @@ -0,0 +1,25 @@ +{ + "schema": [ + { + "name": "PageViews", + "type": "bigint" + }, + { + "name": "WindowClientWidth", + "type": "smallint" + }, + { + "name": "WindowClientHeight", + "type": "smallint" + } + ], + "datarows": [ + [ + 1, + 1280, + 1024 + ] + ], + "total": 1, + "size": 1 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q43.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q43.json new file mode 100644 index 00000000000..3450372b184 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q43.json @@ -0,0 +1,32 @@ +{ + "schema": [ + { + "name": "PageViews", + "type": "bigint" + }, + { + "name": "M", + "type": "string" + } + ], + "datarows": [ + [ + 1, + "2013-07-05 11:00:00" + ], + [ + 1, + "2013-07-10 09:00:00" + ], + [ + 1, + "2013-07-14 14:00:00" + ], + [ + 1, + "2013-07-15 10:00:00" + ] + ], + "total": 4, + "size": 4 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q5.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q5.json new file mode 100644 index 00000000000..60fc2d02469 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q5.json @@ -0,0 +1,15 @@ +{ + "schema": [ + { + "name": "dc(UserID)", + "type": "bigint" + } + ], + "datarows": [ + [ + 8 + ] + ], + "total": 1, + "size": 1 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q6.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q6.json new file mode 100644 index 00000000000..993da52a412 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q6.json @@ -0,0 +1,15 @@ +{ + "schema": [ + { + "name": "dc(SearchPhrase)", + "type": "bigint" + } + ], + "datarows": [ + [ + 7 + ] + ], + "total": 1, + "size": 1 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q7.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q7.json new file mode 100644 index 00000000000..d5d4fdf29e3 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q7.json @@ -0,0 +1,20 @@ +{ + "schema": [ + { + "name": "min(EventDate)", + "type": "timestamp" + }, + { + "name": "max(EventDate)", + "type": "timestamp" + } + ], + "datarows": [ + [ + "2013-07-01 00:00:00", + "2013-07-22 00:00:00" + ] + ], + "total": 1, + "size": 1 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q8.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q8.json new file mode 100644 index 00000000000..ba7eb927d4e --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q8.json @@ -0,0 +1,28 @@ +{ + "schema": [ + { + "name": "count()", + "type": "bigint" + }, + { + "name": "AdvEngineID", + "type": "smallint" + } + ], + "datarows": [ + [ + 3, + 2 + ], + [ + 1, + 1 + ], + [ + 1, + 4 + ] + ], + "total": 3, + "size": 3 +} diff --git a/integ-test/src/test/resources/clickbench/queries/expected/expected-q9.json b/integ-test/src/test/resources/clickbench/queries/expected/expected-q9.json new file mode 100644 index 00000000000..4bff669b957 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/expected/expected-q9.json @@ -0,0 +1,44 @@ +{ + "schema": [ + { + "name": "u", + "type": "bigint" + }, + { + "name": "RegionID", + "type": "int" + } + ], + "datarows": [ + [ + 2, + 104 + ], + [ + 1, + 100 + ], + [ + 1, + 101 + ], + [ + 1, + 102 + ], + [ + 1, + 103 + ], + [ + 1, + 331 + ], + [ + 1, + 479 + ] + ], + "total": 7, + "size": 7 +} diff --git a/integ-test/src/test/resources/clickbench/queries/q10.ppl b/integ-test/src/test/resources/clickbench/queries/q10.ppl index 11ac3b319db..3756c48afdf 100644 --- a/integ-test/src/test/resources/clickbench/queries/q10.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q10.ppl @@ -3,6 +3,6 @@ SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DI FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10; */ source=hits -| stats sum(AdvEngineID), count() as c, avg(ResolutionWidth), dc(UserID) by RegionID +| stats bucket_nullable=false sum(AdvEngineID), count() as c, avg(ResolutionWidth), dc(UserID) by RegionID | sort - c -| head 10 \ No newline at end of file +| head 10 diff --git a/integ-test/src/test/resources/clickbench/queries/q11.ppl b/integ-test/src/test/resources/clickbench/queries/q11.ppl index 1df76937bf3..d426b7e49d7 100644 --- a/integ-test/src/test/resources/clickbench/queries/q11.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q11.ppl @@ -5,6 +5,6 @@ GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10; */ source=hits | where MobilePhoneModel != '' -| stats dc(UserID) as u by MobilePhoneModel +| stats bucket_nullable=false dc(UserID) as u by MobilePhoneModel | sort - u -| head 10 \ No newline at end of file +| head 10 diff --git a/integ-test/src/test/resources/clickbench/queries/q12.ppl b/integ-test/src/test/resources/clickbench/queries/q12.ppl index fd78378c362..93839d8ca2a 100644 --- a/integ-test/src/test/resources/clickbench/queries/q12.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q12.ppl @@ -5,6 +5,6 @@ GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10; */ source=hits | where MobilePhoneModel != '' -| stats dc(UserID) as u by MobilePhone, MobilePhoneModel +| stats bucket_nullable=false dc(UserID) as u by MobilePhone, MobilePhoneModel | sort - u -| head 10 \ No newline at end of file +| head 10 diff --git a/integ-test/src/test/resources/clickbench/queries/q13.ppl b/integ-test/src/test/resources/clickbench/queries/q13.ppl index deaad26c24b..512684207ff 100644 --- a/integ-test/src/test/resources/clickbench/queries/q13.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q13.ppl @@ -4,6 +4,6 @@ GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; */ source=hits | where SearchPhrase != '' -| stats count() as c by SearchPhrase +| stats bucket_nullable=false count() as c by SearchPhrase | sort - c -| head 10 \ No newline at end of file +| head 10 diff --git a/integ-test/src/test/resources/clickbench/queries/q14.ppl b/integ-test/src/test/resources/clickbench/queries/q14.ppl index 80b896bdbd4..3019ed6642e 100644 --- a/integ-test/src/test/resources/clickbench/queries/q14.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q14.ppl @@ -5,6 +5,6 @@ GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10; */ source=hits | where SearchPhrase != '' -| stats dc(UserID) as u by SearchPhrase +| stats bucket_nullable=false dc(UserID) as u by SearchPhrase | sort - u -| head 10 \ No newline at end of file +| head 10 diff --git a/integ-test/src/test/resources/clickbench/queries/q15.ppl b/integ-test/src/test/resources/clickbench/queries/q15.ppl index 44cbd81b0ab..8776f0704dc 100644 --- a/integ-test/src/test/resources/clickbench/queries/q15.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q15.ppl @@ -5,6 +5,6 @@ GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10; */ source=hits | where SearchPhrase != '' -| stats count() as c by SearchEngineID, SearchPhrase +| stats bucket_nullable=false count() as c by SearchEngineID, SearchPhrase | sort - c -| head 10 \ No newline at end of file +| head 10 diff --git a/integ-test/src/test/resources/clickbench/queries/q16.ppl b/integ-test/src/test/resources/clickbench/queries/q16.ppl index 157febefec4..a7ba24bb1cf 100644 --- a/integ-test/src/test/resources/clickbench/queries/q16.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q16.ppl @@ -2,6 +2,6 @@ SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10; */ source=hits -| stats count() by UserID +| stats bucket_nullable=false count() by UserID | sort - `count()` -| head 10 \ No newline at end of file +| head 10 diff --git a/integ-test/src/test/resources/clickbench/queries/q17.ppl b/integ-test/src/test/resources/clickbench/queries/q17.ppl index 3dfc82c236a..7c88bc56bd4 100644 --- a/integ-test/src/test/resources/clickbench/queries/q17.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q17.ppl @@ -3,6 +3,6 @@ SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; */ source=hits -| stats count() by UserID, SearchPhrase +| stats bucket_nullable=false count() by UserID, SearchPhrase | sort - `count()` -| head 10 \ No newline at end of file +| head 10 diff --git a/integ-test/src/test/resources/clickbench/queries/q18.ppl b/integ-test/src/test/resources/clickbench/queries/q18.ppl index 38b77a5a565..e66b4e11ef3 100644 --- a/integ-test/src/test/resources/clickbench/queries/q18.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q18.ppl @@ -2,5 +2,5 @@ SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10; */ source=hits -| stats count() by UserID, SearchPhrase -| head 10 \ No newline at end of file +| stats bucket_nullable=false count() by UserID, SearchPhrase +| head 10 diff --git a/integ-test/src/test/resources/clickbench/queries/q19.ppl b/integ-test/src/test/resources/clickbench/queries/q19.ppl index edd852f528b..875ba585c08 100644 --- a/integ-test/src/test/resources/clickbench/queries/q19.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q19.ppl @@ -4,6 +4,6 @@ FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; */ source=hits | eval m = extract(minute from EventTime) -| stats count() by UserID, m, SearchPhrase +| stats bucket_nullable=false count() by UserID, m, SearchPhrase | sort - `count()` -| head 10 \ No newline at end of file +| head 10 diff --git a/integ-test/src/test/resources/clickbench/queries/q22.ppl b/integ-test/src/test/resources/clickbench/queries/q22.ppl index 3319cac2a9b..b4f51f40b21 100644 --- a/integ-test/src/test/resources/clickbench/queries/q22.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q22.ppl @@ -5,6 +5,6 @@ GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; */ source=hits | where like(URL, '%google%') and SearchPhrase != '' -| stats /* min(URL), */ count() as c by SearchPhrase +| stats bucket_nullable=false /* min(URL), */ count() as c by SearchPhrase | sort - c -| head 10 \ No newline at end of file +| head 10 diff --git a/integ-test/src/test/resources/clickbench/queries/q23.ppl b/integ-test/src/test/resources/clickbench/queries/q23.ppl index d5c6de41cec..eb5ed5b0102 100644 --- a/integ-test/src/test/resources/clickbench/queries/q23.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q23.ppl @@ -5,6 +5,6 @@ GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; */ source=hits | where like(Title, '%Google%') and not like(URL, '%.google.%') and SearchPhrase != '' -| stats /* min(URL), min(Title), */ count() as c, dc(UserID) by SearchPhrase +| stats bucket_nullable=false /* min(URL), min(Title), */ count() as c, dc(UserID) by SearchPhrase | sort - c -| head 10 \ No newline at end of file +| head 10 diff --git a/integ-test/src/test/resources/clickbench/queries/q28.ppl b/integ-test/src/test/resources/clickbench/queries/q28.ppl index 925b19fb328..3d9cbbcc6b6 100644 --- a/integ-test/src/test/resources/clickbench/queries/q28.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q28.ppl @@ -4,7 +4,7 @@ FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l */ source=hits | where URL != '' -| stats avg(length(URL)) as l, count() as c by CounterID -| where c > 100000 +| stats bucket_nullable=false avg(length(URL)) as l, count() as c by CounterID +| where c > 1 | sort - l -| head 25 \ No newline at end of file +| head 25 diff --git a/integ-test/src/test/resources/clickbench/queries/q29.ppl b/integ-test/src/test/resources/clickbench/queries/q29.ppl index 5de7dea5ff0..5d46bef42f8 100644 --- a/integ-test/src/test/resources/clickbench/queries/q29.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q29.ppl @@ -6,7 +6,7 @@ FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DES source=hits | Referer != '' | eval k = regexp_replace(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') -| stats avg(length(Referer)) as l, count() as c, min(Referer) by k -| where c > 100000 +| stats bucket_nullable=false avg(length(Referer)) as l, count() as c, min(Referer) by k +| where c > 1 | sort - l -| head 25 \ No newline at end of file +| head 25 diff --git a/integ-test/src/test/resources/clickbench/queries/q31.ppl b/integ-test/src/test/resources/clickbench/queries/q31.ppl index 1cff19bac32..ec784134363 100644 --- a/integ-test/src/test/resources/clickbench/queries/q31.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q31.ppl @@ -4,6 +4,6 @@ FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c */ source=hits | where SearchPhrase != '' -| stats count() as c, sum(IsRefresh), avg(ResolutionWidth) by SearchEngineID, ClientIP +| stats bucket_nullable=false count() as c, sum(IsRefresh), avg(ResolutionWidth) by SearchEngineID, ClientIP | sort - c -| head 10 \ No newline at end of file +| head 10 diff --git a/integ-test/src/test/resources/clickbench/queries/q32.ppl b/integ-test/src/test/resources/clickbench/queries/q32.ppl index 1a9c7214048..7465fe2abe2 100644 --- a/integ-test/src/test/resources/clickbench/queries/q32.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q32.ppl @@ -4,6 +4,6 @@ FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LI */ source=hits | where SearchPhrase != '' -| stats count() as c, sum(IsRefresh), avg(ResolutionWidth) by WatchID, ClientIP +| stats bucket_nullable=false count() as c, sum(IsRefresh), avg(ResolutionWidth) by WatchID, ClientIP | sort - c -| head 10 \ No newline at end of file +| head 10 diff --git a/integ-test/src/test/resources/clickbench/queries/q33.ppl b/integ-test/src/test/resources/clickbench/queries/q33.ppl index 06ff6329889..55d8a0d9bb7 100644 --- a/integ-test/src/test/resources/clickbench/queries/q33.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q33.ppl @@ -3,6 +3,6 @@ SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; */ source=hits -| stats count() as c, sum(IsRefresh), avg(ResolutionWidth) by WatchID, ClientIP +| stats bucket_nullable=false count() as c, sum(IsRefresh), avg(ResolutionWidth) by WatchID, ClientIP | sort - c -| head 10 \ No newline at end of file +| head 10 diff --git a/integ-test/src/test/resources/clickbench/queries/q34.ppl b/integ-test/src/test/resources/clickbench/queries/q34.ppl index b6813b38db6..7653ab48a8e 100644 --- a/integ-test/src/test/resources/clickbench/queries/q34.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q34.ppl @@ -2,6 +2,6 @@ SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10; */ source=hits -| stats count() as c by URL +| stats bucket_nullable=false count() as c by URL | sort - c -| head 10 \ No newline at end of file +| head 10 diff --git a/integ-test/src/test/resources/clickbench/queries/q35.ppl b/integ-test/src/test/resources/clickbench/queries/q35.ppl index e52640b9dbf..a510cde803e 100644 --- a/integ-test/src/test/resources/clickbench/queries/q35.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q35.ppl @@ -3,6 +3,6 @@ SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10; */ source=hits | eval const = 1 -| stats count() as c by const, URL +| stats bucket_nullable=false count() as c by const, URL | sort - c -| head 10 \ No newline at end of file +| head 10 diff --git a/integ-test/src/test/resources/clickbench/queries/q36.ppl b/integ-test/src/test/resources/clickbench/queries/q36.ppl index 78f89060945..b5c7a1faf98 100644 --- a/integ-test/src/test/resources/clickbench/queries/q36.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q36.ppl @@ -4,6 +4,6 @@ FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c */ source=hits | eval `ClientIP - 1` = ClientIP - 1, `ClientIP - 2` = ClientIP - 2, `ClientIP - 3` = ClientIP - 3 -| stats count() as c by `ClientIP`, `ClientIP - 1`, `ClientIP - 2`, `ClientIP - 3` +| stats bucket_nullable=false count() as c by `ClientIP`, `ClientIP - 1`, `ClientIP - 2`, `ClientIP - 3` | sort - c -| head 10 \ No newline at end of file +| head 10 diff --git a/integ-test/src/test/resources/clickbench/queries/q37.ppl b/integ-test/src/test/resources/clickbench/queries/q37.ppl index 7fe4dc49486..d829077afdb 100644 --- a/integ-test/src/test/resources/clickbench/queries/q37.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q37.ppl @@ -6,6 +6,6 @@ GROUP BY URL ORDER BY PageViews DESC LIMIT 10; */ source=hits | where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and DontCountHits = 0 and IsRefresh = 0 and URL != '' -| stats count() as PageViews by URL +| stats bucket_nullable=false count() as PageViews by URL | sort - PageViews -| head 10 \ No newline at end of file +| head 10 diff --git a/integ-test/src/test/resources/clickbench/queries/q38.ppl b/integ-test/src/test/resources/clickbench/queries/q38.ppl index becd5a49a91..53a56cee87b 100644 --- a/integ-test/src/test/resources/clickbench/queries/q38.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q38.ppl @@ -6,6 +6,6 @@ GROUP BY Title ORDER BY PageViews DESC LIMIT 10; */ source=hits | where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and DontCountHits = 0 and IsRefresh = 0 and Title != '' -| stats count() as PageViews by Title +| stats bucket_nullable=false count() as PageViews by Title | sort - PageViews -| head 10 \ No newline at end of file +| head 10 diff --git a/integ-test/src/test/resources/clickbench/queries/q39.ppl b/integ-test/src/test/resources/clickbench/queries/q39.ppl index 141707fc0a9..d3a76649372 100644 --- a/integ-test/src/test/resources/clickbench/queries/q39.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q39.ppl @@ -6,6 +6,6 @@ GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; */ source=hits | where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and IsRefresh = 0 and IsLink != 0 and IsDownload = 0 -| stats count() as PageViews by URL +| stats bucket_nullable=false count() as PageViews by URL | sort - PageViews -| head 10 from 1000 \ No newline at end of file +| head 10 from 1 diff --git a/integ-test/src/test/resources/clickbench/queries/q40.ppl b/integ-test/src/test/resources/clickbench/queries/q40.ppl index a3481eb2fac..22ceb36c75d 100644 --- a/integ-test/src/test/resources/clickbench/queries/q40.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q40.ppl @@ -6,6 +6,6 @@ GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageView source=hits | where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and IsRefresh = 0 | eval Src=case(SearchEngineID = 0 and AdvEngineID = 0, Referer else ''), Dst=URL -| stats count() as PageViews by TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst +| stats /*bucket_nullable=false*/ count() as PageViews by TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst | sort - PageViews -| head 10 from 1000 \ No newline at end of file +| head 10 from 1 diff --git a/integ-test/src/test/resources/clickbench/queries/q41.ppl b/integ-test/src/test/resources/clickbench/queries/q41.ppl index 13028d744bc..8cde43b3152 100644 --- a/integ-test/src/test/resources/clickbench/queries/q41.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q41.ppl @@ -6,6 +6,6 @@ GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100; */ source=hits | where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and IsRefresh = 0 and TraficSourceID in (-1, 6) and RefererHash = 3594120000172545465 -| stats count() as PageViews by URLHash, EventDate +| stats bucket_nullable=false count() as PageViews by URLHash, EventDate | sort - PageViews -| head 10 from 100 \ No newline at end of file +| head 10 from 100 diff --git a/integ-test/src/test/resources/clickbench/queries/q42.ppl b/integ-test/src/test/resources/clickbench/queries/q42.ppl index 0d5f034c129..a4a6a740e6d 100644 --- a/integ-test/src/test/resources/clickbench/queries/q42.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q42.ppl @@ -6,6 +6,6 @@ GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 */ source=hits | where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and IsRefresh = 0 and DontCountHits = 0 and URLHash = 2868770270353813622 -| stats count() as PageViews by WindowClientWidth, WindowClientHeight +| stats bucket_nullable=false count() as PageViews by WindowClientWidth, WindowClientHeight | sort - PageViews -| head 10 from 10000 \ No newline at end of file +| head 10 from 1 diff --git a/integ-test/src/test/resources/clickbench/queries/q43.ppl b/integ-test/src/test/resources/clickbench/queries/q43.ppl index b0a697a8367..9e0d7a5ce94 100644 --- a/integ-test/src/test/resources/clickbench/queries/q43.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q43.ppl @@ -9,6 +9,6 @@ LIMIT 10 OFFSET 1000; source=hits | where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-15 00:00:00' and IsRefresh = 0 and DontCountHits = 0 | eval M = date_format(EventTime, '%Y-%m-%d %H:00:00') -| stats count() as PageViews by M +| stats /*bucket_nullable=false*/ count() as PageViews by M | sort M -| head 10 from 1000 \ No newline at end of file +| head 10 from 1 diff --git a/integ-test/src/test/resources/clickbench/queries/q8.ppl b/integ-test/src/test/resources/clickbench/queries/q8.ppl index b61f73a805b..99aed6ad25b 100644 --- a/integ-test/src/test/resources/clickbench/queries/q8.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q8.ppl @@ -1,4 +1,4 @@ /* SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC; */ -source=hits | where AdvEngineID!=0 | stats count() by AdvEngineID | sort - `count()` \ No newline at end of file +source=hits | where AdvEngineID!=0 | stats bucket_nullable=false count() by AdvEngineID | sort - `count()` diff --git a/integ-test/src/test/resources/clickbench/queries/q9.ppl b/integ-test/src/test/resources/clickbench/queries/q9.ppl index aa6f08f8ce0..3abd8ae229a 100644 --- a/integ-test/src/test/resources/clickbench/queries/q9.ppl +++ b/integ-test/src/test/resources/clickbench/queries/q9.ppl @@ -1,4 +1,4 @@ /* SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10; */ -source=hits | stats dc(UserID) as u by RegionID | sort -u | head 10 \ No newline at end of file +source=hits | stats bucket_nullable=false dc(UserID) as u by RegionID | sort -u | head 10 diff --git a/integ-test/src/test/resources/events_traffic.json b/integ-test/src/test/resources/events_traffic.json new file mode 100644 index 00000000000..bcb3fe17f2a --- /dev/null +++ b/integ-test/src/test/resources/events_traffic.json @@ -0,0 +1,12 @@ +{"index":{"_id":"1"}} +{"@timestamp":"2025-09-08T10:00:00","packets":60,"host":"server1"} +{"index":{"_id":"2"}} +{"@timestamp":"2025-09-08T10:01:00","packets":120,"host":"server1"} +{"index":{"_id":"3"}} +{"@timestamp":"2025-09-08T10:02:00","packets":60,"host":"server1"} +{"index":{"_id":"4"}} +{"@timestamp":"2025-09-08T10:02:30","packets":180,"host":"server2"} +{"index":{"_id":"5"}} +{"@timestamp":"2025-02-15T14:00:00","packets":18748800,"host":"server1"} +{"index":{"_id":"6"}} +{"@timestamp":"2025-10-15T14:00:00","packets":18748800,"host":"server1"} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/access_struct_subfield_with_item.yaml b/integ-test/src/test/resources/expectedOutput/calcite/access_struct_subfield_with_item.yaml new file mode 100644 index 00000000000..a3726ad6126 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/access_struct_subfield_with_item.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(host=[$0], info=[GEOIP('dummy-datasource':VARCHAR, $0)], info.dummy_sub_field=[ITEM(GEOIP('dummy-datasource':VARCHAR, $0), 'dummy_sub_field')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) + physical: | + EnumerableCalc(expr#0=[{inputs}], expr#1=['dummy-datasource':VARCHAR], expr#2=[GEOIP($t1, $t0)], expr#3=['dummy_sub_field'], expr#4=[ITEM($t2, $t3)], host=[$t0], $f1=[$t2], $f2=[$t4]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[PROJECT->[host], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["host"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_case_cannot_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_case_cannot_push.yaml new file mode 100644 index 00000000000..d04bbd2df44 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_case_cannot_push.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg_age=[$1], age_range=[$0]) + LogicalAggregate(group=[{0}], avg_age=[AVG($1)]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, SEARCH($10, Sarg[[30..40]]), 'u40':VARCHAR, 'u100':VARCHAR)], age=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg_age=AVG($1)), PROJECT->[avg_age, age_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age_range":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BXZ7CiAgIm9wIjogewogICAgIm5hbWUiOiAiQ0FTRSIsCiAgICAia2luZCI6ICJDQVNFIiwKICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIjwiLAogICAgICAgICJraW5kIjogIkxFU1NfVEhBTiIsCiAgICAgICAgInN5bnRheCI6ICJCSU5BUlkiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgIm5hbWUiOiAiJDAiCiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAibGl0ZXJhbCI6IDMwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAidTMwIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJTRUFSQ0giLAogICAgICAgICJraW5kIjogIlNFQVJDSCIsCiAgICAgICAgInN5bnRheCI6ICJJTlRFUk5BTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogewogICAgICAgICAgICAicmFuZ2VTZXQiOiBbCiAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgImNsb3NlZCIsCiAgICAgICAgICAgICAgICAiMzAiLAogICAgICAgICAgICAgICAgIjQwIgogICAgICAgICAgICAgIF0KICAgICAgICAgICAgXSwKICAgICAgICAgICAgIm51bGxBcyI6ICJVTktOT1dOIgogICAgICAgICAgfSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogInU0MCIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICJ1MTAwIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_case_composite_cannot_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_case_composite_cannot_push.yaml new file mode 100644 index 00000000000..82cbadeb735 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_case_composite_cannot_push.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg_balance=[$2], age_range=[$0], state=[$1]) + LogicalAggregate(group=[{0, 1}], avg_balance=[AVG($2)]) + LogicalProject(age_range=[CASE(<($10, 35), 'u35':VARCHAR, $11)], state=[$9], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2)), PROJECT->[avg_balance, age_range, state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age_range":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQA5nsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQCe3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiPCIsCiAgICAgICAgImtpbmQiOiAiTEVTU19USEFOIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogMzUsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICJ1MzUiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImlucHV0IjogMSwKICAgICAgIm5hbWUiOiAiJDEiCiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAACdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnQABWVtYWlsc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+ABB4cH5xAH4ACnQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgALdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AG3hwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AHQAAAABzcQB+AAAAAAADdwQAAAAAeHh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_case_num_res_cannot_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_case_num_res_cannot_push.yaml new file mode 100644 index 00000000000..9502c66a448 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_case_num_res_cannot_push.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], age_range=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(age_range=[CASE(<($10, 30), 30, 100)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), age_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age_range":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0Ap17CiAgIm9wIjogewogICAgIm5hbWUiOiAiQ0FTRSIsCiAgICAia2luZCI6ICJDQVNFIiwKICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIjwiLAogICAgICAgICJraW5kIjogIkxFU1NfVEhBTiIsCiAgICAgICAgInN5bnRheCI6ICJCSU5BUlkiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgIm5hbWUiOiAiJDAiCiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAibGl0ZXJhbCI6IDMwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAzMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAxMDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":true,"value_type":"long","missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite2_range_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite2_range_count_push.yaml new file mode 100644 index 00000000000..353bcf5c1e9 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite2_range_count_push.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg(balance)=[$3], count()=[$4], age_range=[$0], state=[$1], gender=[$2]) + LogicalAggregate(group=[{0, 1, 2}], avg(balance)=[AVG($3)], count()=[COUNT()]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, 'a30':VARCHAR)], state=[$9], gender=[$4], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2},avg(balance)=AVG($3),count()=COUNT()), PROJECT->[avg(balance), count(), age_range, state, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u30","to":30.0},{"key":"a30","from":30.0}],"keyed":true},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite2_range_range_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite2_range_range_count_push.yaml new file mode 100644 index 00000000000..eef2a7b23f8 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite2_range_range_count_push.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg_balance=[$3], age_range=[$0], balance_range=[$1], state=[$2]) + LogicalAggregate(group=[{0, 1, 2}], avg_balance=[AVG($3)]) + LogicalProject(age_range=[CASE(<($10, 35), 'u35':VARCHAR, 'a35':VARCHAR)], balance_range=[CASE(<($7, 20000), 'medium':VARCHAR, 'high':VARCHAR)], state=[$9], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2},avg_balance=AVG($3)), PROJECT->[avg_balance, age_range, balance_range, state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u35","to":35.0},{"key":"a35","from":35.0}],"keyed":true},"aggregations":{"balance_range":{"range":{"field":"balance","ranges":[{"key":"medium","to":20000.0},{"key":"high","from":20000.0}],"keyed":true},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_autodate_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_autodate_range_metric_push.yaml new file mode 100644 index 00000000000..dccce23e18b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_autodate_range_metric_push.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg(value)=[$3], count()=[$4], timestamp=[$0], value_range=[$1], category=[$2]) + LogicalAggregate(group=[{0, 1, 2}], avg(value)=[AVG($3)], count()=[COUNT()]) + LogicalProject(timestamp=[$9], value_range=[$10], category=[$1], value=[$2]) + LogicalFilter(condition=[AND(IS NOT NULL($9), IS NOT NULL($1))]) + LogicalProject(@timestamp=[$0], category=[$1], value=[$2], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9], timestamp=[WIDTH_BUCKET($3, 3, -(MAX($3) OVER (), MIN($3) OVER ()), MAX($3) OVER ())], value_range=[CASE(<($2, 7000), 'small':VARCHAR, 'great':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2, 3},avg(value)=AVG($1),count()=COUNT()), PROJECT->[avg(value), count(), timestamp, value_range, category], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"timestamp":{"auto_date_histogram":{"field":"timestamp","buckets":3,"minimum_interval":null},"aggregations":{"value_range":{"range":{"field":"value","ranges":[{"key":"small","to":7000.0},{"key":"great","from":7000.0}],"keyed":true},"aggregations":{"avg(value)":{"avg":{"field":"value"}}}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_autodate_range_metric_sort_agg_measure_not_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_autodate_range_metric_sort_agg_measure_not_push.yaml new file mode 100644 index 00000000000..90e83946c38 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_autodate_range_metric_sort_agg_measure_not_push.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) + LogicalProject(avg(value)=[$3], cnt=[$4], timestamp=[$0], value_range=[$1], category=[$2]) + LogicalAggregate(group=[{0, 1, 2}], avg(value)=[AVG($3)], cnt=[COUNT()]) + LogicalProject(timestamp=[$9], value_range=[$10], category=[$1], value=[$2]) + LogicalFilter(condition=[AND(IS NOT NULL($9), IS NOT NULL($1))]) + LogicalProject(@timestamp=[$0], category=[$1], value=[$2], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9], timestamp=[WIDTH_BUCKET($3, 3, -(MAX($3) OVER (), MIN($3) OVER ()), MAX($3) OVER ())], value_range=[CASE(<($2, 7000), 'small':VARCHAR, 'great':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2, 3},avg(value)=AVG($1),cnt=COUNT()), PROJECT->[avg(value), cnt, timestamp, value_range, category]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"timestamp":{"auto_date_histogram":{"field":"timestamp","buckets":3,"minimum_interval":null},"aggregations":{"value_range":{"range":{"field":"value","ranges":[{"key":"small","to":7000.0},{"key":"great","from":7000.0}],"keyed":true},"aggregations":{"avg(value)":{"avg":{"field":"value"}}}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_autodate_sort_agg_measure_not_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_autodate_sort_agg_measure_not_push.yaml new file mode 100644 index 00000000000..e3d4d9fba4d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_autodate_sort_agg_measure_not_push.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) + LogicalProject(avg(value)=[$2], cnt=[$3], timestamp=[$0], category=[$1]) + LogicalAggregate(group=[{0, 1}], avg(value)=[AVG($2)], cnt=[COUNT()]) + LogicalProject(timestamp=[$9], category=[$1], value=[$2]) + LogicalFilter(condition=[AND(IS NOT NULL($9), IS NOT NULL($1))]) + LogicalProject(@timestamp=[$0], category=[$1], value=[$2], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9], timestamp=[WIDTH_BUCKET($3, 3, -(MAX($3) OVER (), MIN($3) OVER ()), MAX($3) OVER ())]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(value)=AVG($1),cnt=COUNT()), PROJECT->[avg(value), cnt, timestamp, category]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"timestamp":{"auto_date_histogram":{"field":"timestamp","buckets":3,"minimum_interval":null},"aggregations":{"avg(value)":{"avg":{"field":"value"}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_date_range_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_date_range_push.yaml new file mode 100644 index 00000000000..30e4762d325 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_date_range_push.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg(value)=[$2], span(@timestamp,1h)=[$1], value_range=[$0]) + LogicalAggregate(group=[{0, 2}], avg(value)=[AVG($1)]) + LogicalProject(value_range=[$10], value=[$2], span(@timestamp,1h)=[SPAN($0, 1, 'h')]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9], value_range=[CASE(<($2, 7000), 'small':VARCHAR, 'large':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(value)=AVG($1)), PROJECT->[avg(value), span(@timestamp,1h), value_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"@timestamp","boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(@timestamp,1h)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1h"}}}]},"aggregations":{"value_range":{"range":{"field":"value","ranges":[{"key":"small","to":7000.0},{"key":"large","from":7000.0}],"keyed":true},"aggregations":{"avg(value)":{"avg":{"field":"value"}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_multi_terms_autodate_sort_agg_measure_not_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_multi_terms_autodate_sort_agg_measure_not_push.yaml new file mode 100644 index 00000000000..6995097f878 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_multi_terms_autodate_sort_agg_measure_not_push.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) + LogicalProject(avg(value)=[$3], cnt=[$4], category=[$0], value=[$1], timestamp=[$2]) + LogicalAggregate(group=[{0, 1, 2}], avg(value)=[AVG($1)], cnt=[COUNT()]) + LogicalProject(category=[$1], value=[$2], timestamp=[$9]) + LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($2), IS NOT NULL($9))]) + LogicalProject(@timestamp=[$0], category=[$1], value=[$2], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9], timestamp=[WIDTH_BUCKET($3, 3, -(MAX($3) OVER (), MIN($3) OVER ()), MAX($3) OVER ())]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[CAST($t1):DOUBLE], avg(value)=[$t4], cnt=[$t3], category=[$t0], value=[$t1], timestamp=[$t2]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2},cnt=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":false,"order":"asc"}}},{"value":{"terms":{"field":"value","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"timestamp":{"auto_date_histogram":{"field":"timestamp","buckets":3,"minimum_interval":null}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_range_metric_push.yaml new file mode 100644 index 00000000000..065598bc82c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_range_metric_push.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg(balance)=[$2], state=[$0], age_range=[$1]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(state=[$9], age_range=[CASE(<($10, 30), 'u30':VARCHAR, 'a30':VARCHAR)], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), PROJECT->[avg(balance), state, age_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u30","to":30.0},{"key":"a30","from":30.0}],"keyed":true},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_range_sort_agg_measure_not_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_range_sort_agg_measure_not_push.yaml new file mode 100644 index 00000000000..19846e9910b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_range_sort_agg_measure_not_push.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) + LogicalProject(avg(value)=[$2], cnt=[$3], value_range=[$0], category=[$1]) + LogicalAggregate(group=[{0, 1}], avg(value)=[AVG($2)], cnt=[COUNT()]) + LogicalProject(value_range=[$10], category=[$1], value=[$2]) + LogicalFilter(condition=[IS NOT NULL($1)]) + LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9], value_range=[CASE(<($2, 7000), 'small':VARCHAR, 'great':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(value)=AVG($1),cnt=COUNT()), PROJECT->[avg(value), cnt, value_range, category]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"value_range":{"range":{"field":"value","ranges":[{"key":"small","to":7000.0},{"key":"great","from":7000.0}],"keyed":true},"aggregations":{"avg(value)":{"avg":{"field":"value"}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_range_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_range_count_push.yaml new file mode 100644 index 00000000000..498786a6aef --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_range_count_push.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg(age)=[$1], age_range=[$0]) + LogicalAggregate(group=[{0}], avg(age)=[AVG($1)]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, SEARCH($10, Sarg[[30..40)]), 'u40':VARCHAR, 'u100':VARCHAR)], age=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(age)=AVG($1)), PROJECT->[avg(age), age_range]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u30","to":30.0},{"key":"u40","from":30.0,"to":40.0},{"key":"u100","from":40.0}],"keyed":true},"aggregations":{"avg(age)":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_range_metric_complex_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_range_metric_complex_push.yaml new file mode 100644 index 00000000000..f3d749487c0 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_range_metric_complex_push.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg(balance)=[$1], age_range=[$0]) + LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, SEARCH($10, Sarg[[35..40), [80..+∞)]), '30-40 or >=80':VARCHAR, null:NULL)], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(balance)=AVG($1)), PROJECT->[avg(balance), age_range]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u30","to":30.0},{"key":"30-40 or >=80","from":35.0,"to":40.0},{"key":"30-40 or >=80","from":80.0},{"key":"null","from":30.0,"to":35.0},{"key":"null","from":40.0,"to":80.0}],"keyed":true},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_range_metric_push.yaml new file mode 100644 index 00000000000..ee0a5ce9448 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_range_metric_push.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg_age=[$1], age_range=[$0]) + LogicalAggregate(group=[{0}], avg_age=[AVG($1)]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, <($10, 40), 'u40':VARCHAR, 'u100':VARCHAR)], age=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg_age=AVG($1)), PROJECT->[avg_age, age_range]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u30","to":30.0},{"key":"u40","from":30.0,"to":40.0},{"key":"u100","from":40.0}],"keyed":true},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_range_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_range_range_metric_push.yaml new file mode 100644 index 00000000000..5b44ebfdc68 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_range_range_metric_push.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg_balance=[$2], age_range=[$0], balance_range=[$1]) + LogicalAggregate(group=[{0, 1}], avg_balance=[AVG($2)]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, <($10, 40), 'u40':VARCHAR, 'u100':VARCHAR)], balance_range=[CASE(<($7, 20000), 'medium':VARCHAR, 'high':VARCHAR)], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2)), PROJECT->[avg_balance, age_range, balance_range]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u30","to":30.0},{"key":"u40","from":30.0,"to":40.0},{"key":"u100","from":40.0}],"keyed":true},"aggregations":{"balance_range":{"range":{"field":"balance","ranges":[{"key":"medium","to":20000.0},{"key":"high","from":20000.0}],"keyed":true},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_timestamp.yaml new file mode 100644 index 00000000000..81138f6fe80 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_timestamp.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], SORT->[{ + "@timestamp" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_timestamp_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_timestamp_can_match_shortcut.yaml new file mode 100644 index 00000000000..ce84d53f479 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_timestamp_can_match_shortcut.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) + LogicalFilter(condition=[query_string(MAP('query', 'process.name:kernel':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->query_string(MAP('query', 'process.name:kernel':VARCHAR)), SORT->[{ + "@timestamp" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_timestamp_no_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_timestamp_no_can_match_shortcut.yaml new file mode 100644 index 00000000000..ce84d53f479 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_timestamp_no_can_match_shortcut.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) + LogicalFilter(condition=[query_string(MAP('query', 'process.name:kernel':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->query_string(MAP('query', 'process.name:kernel':VARCHAR)), SORT->[{ + "@timestamp" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_with_after_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_with_after_timestamp.yaml new file mode 100644 index 00000000000..81138f6fe80 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_with_after_timestamp.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], SORT->[{ + "@timestamp" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/cardinality_agg_high.yaml b/integ-test/src/test/resources/expectedOutput/calcite/cardinality_agg_high.yaml new file mode 100644 index 00000000000..bd3889e2926 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/cardinality_agg_high.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], dc(`agent.name`)=[COUNT(DISTINCT $0)]) + LogicalProject(agent.name=[$3]) + LogicalFilter(condition=[IS NOT NULL($3)]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[FILTER->IS NOT NULL($3), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},dc(`agent.name`)=COUNT(DISTINCT $0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"agent.name","boost":1.0}},"aggregations":{"dc(`agent.name`)":{"cardinality":{"field":"agent.name"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/cardinality_agg_high_2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/cardinality_agg_high_2.yaml new file mode 100644 index 00000000000..6d5c2b7448e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/cardinality_agg_high_2.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], dc(`event.id`)=[COUNT(DISTINCT $0)]) + LogicalProject(event.id=[$37]) + LogicalFilter(condition=[IS NOT NULL($37)]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[FILTER->IS NOT NULL($37), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},dc(`event.id`)=COUNT(DISTINCT $0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"event.id","boost":1.0}},"aggregations":{"dc(`event.id`)":{"cardinality":{"field":"event.id"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/cardinality_agg_low.yaml b/integ-test/src/test/resources/expectedOutput/calcite/cardinality_agg_low.yaml new file mode 100644 index 00000000000..dec25a78628 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/cardinality_agg_low.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], dc(`cloud.region`)=[COUNT(DISTINCT $0)]) + LogicalProject(cloud.region=[$14]) + LogicalFilter(condition=[IS NOT NULL($14)]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[FILTER->IS NOT NULL($14), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},dc(`cloud.region`)=COUNT(DISTINCT $0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"cloud.region","boost":1.0}},"aggregations":{"dc(`cloud.region`)":{"cardinality":{"field":"cloud.region"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q1.yaml new file mode 100644 index 00000000000..f5d0ea7692b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q1.yaml @@ -0,0 +1,7 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], count()=[COUNT()]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"count()":{"value_count":{"field":"_index"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q10.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q10.yaml new file mode 100644 index 00000000000..8138d506a93 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q10.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$1], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$1], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(sum(AdvEngineID)=[$1], c=[$2], avg(ResolutionWidth)=[$3], dc(UserID)=[$4], RegionID=[$0]) + LogicalAggregate(group=[{0}], sum(AdvEngineID)=[SUM($1)], c=[COUNT()], avg(ResolutionWidth)=[AVG($2)], dc(UserID)=[COUNT(DISTINCT $3)]) + LogicalProject(RegionID=[$68], AdvEngineID=[$19], ResolutionWidth=[$80], UserID=[$84]) + LogicalFilter(condition=[IS NOT NULL($68)]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableLimit(fetch=[10]) + EnumerableSort(sort0=[$1], dir0=[DESC-nulls-last]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum(AdvEngineID)=SUM($0),c=COUNT(),avg(ResolutionWidth)=AVG($2),dc(UserID)=COUNT(DISTINCT $3)), PROJECT->[sum(AdvEngineID), c, avg(ResolutionWidth), dc(UserID), RegionID]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"RegionID":{"terms":{"field":"RegionID","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"sum(AdvEngineID)":{"sum":{"field":"AdvEngineID"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"dc(UserID)":{"cardinality":{"field":"UserID"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q11.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q11.yaml new file mode 100644 index 00000000000..f21f57a583e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q11.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(u=[$1], MobilePhoneModel=[$0]) + LogicalAggregate(group=[{0}], u=[COUNT(DISTINCT $1)]) + LogicalProject(MobilePhoneModel=[$31], UserID=[$84]) + LogicalFilter(condition=[IS NOT NULL($31)]) + LogicalFilter(condition=[<>($31, '')]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER-><>($31, ''), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},u=COUNT(DISTINCT $1)), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[u, MobilePhoneModel], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"MobilePhoneModel","boost":1.0}}],"must_not":[{"term":{"MobilePhoneModel":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"MobilePhoneModel":{"terms":{"field":"MobilePhoneModel","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"u":"desc"},{"_key":"asc"}]},"aggregations":{"u":{"cardinality":{"field":"UserID"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q12.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q12.yaml new file mode 100644 index 00000000000..2193be94c08 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q12.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(u=[$2], MobilePhone=[$0], MobilePhoneModel=[$1]) + LogicalAggregate(group=[{0, 1}], u=[COUNT(DISTINCT $2)]) + LogicalProject(MobilePhone=[$62], MobilePhoneModel=[$31], UserID=[$84]) + LogicalFilter(condition=[AND(IS NOT NULL($62), IS NOT NULL($31))]) + LogicalFilter(condition=[<>($31, '')]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[MobilePhoneModel, MobilePhone, UserID], FILTER->AND(<>($0, ''), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},u=COUNT(DISTINCT $2)), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[u, MobilePhone, MobilePhoneModel], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"MobilePhoneModel","boost":1.0}}],"must_not":[{"term":{"MobilePhoneModel":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"MobilePhone","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["MobilePhoneModel","MobilePhone","UserID"],"excludes":[]},"aggregations":{"MobilePhone|MobilePhoneModel":{"multi_terms":{"terms":[{"field":"MobilePhone"},{"field":"MobilePhoneModel"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]},"aggregations":{"u":{"cardinality":{"field":"UserID"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q13.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q13.yaml new file mode 100644 index 00000000000..b18a08c410a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q13.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(c=[$1], SearchPhrase=[$0]) + LogicalAggregate(group=[{0}], c=[COUNT()]) + LogicalProject(SearchPhrase=[$63]) + LogicalFilter(condition=[IS NOT NULL($63)]) + LogicalFilter(condition=[<>($63, '')]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER-><>($63, ''), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[c, SearchPhrase], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"SearchPhrase":{"terms":{"field":"SearchPhrase","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q14.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q14.yaml new file mode 100644 index 00000000000..aa980934e37 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q14.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(u=[$1], SearchPhrase=[$0]) + LogicalAggregate(group=[{0}], u=[COUNT(DISTINCT $1)]) + LogicalProject(SearchPhrase=[$63], UserID=[$84]) + LogicalFilter(condition=[IS NOT NULL($63)]) + LogicalFilter(condition=[<>($63, '')]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER-><>($63, ''), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},u=COUNT(DISTINCT $1)), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[u, SearchPhrase], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"SearchPhrase":{"terms":{"field":"SearchPhrase","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"u":"desc"},{"_key":"asc"}]},"aggregations":{"u":{"cardinality":{"field":"UserID"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q15.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q15.yaml new file mode 100644 index 00000000000..ae655f0e533 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q15.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(c=[$2], SearchEngineID=[$0], SearchPhrase=[$1]) + LogicalAggregate(group=[{0, 1}], c=[COUNT()]) + LogicalProject(SearchEngineID=[$65], SearchPhrase=[$63]) + LogicalFilter(condition=[AND(IS NOT NULL($65), IS NOT NULL($63))]) + LogicalFilter(condition=[<>($63, '')]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[SearchPhrase, SearchEngineID], FILTER->AND(<>($0, ''), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},c=COUNT()), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[c, SearchEngineID, SearchPhrase], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"SearchEngineID","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["SearchPhrase","SearchEngineID"],"excludes":[]},"aggregations":{"SearchEngineID|SearchPhrase":{"multi_terms":{"terms":[{"field":"SearchEngineID"},{"field":"SearchPhrase"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]},"aggregations":{"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q16.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q16.yaml new file mode 100644 index 00000000000..aad05d10c58 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q16.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(count()=[$1], UserID=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(UserID=[$84]) + LogicalFilter(condition=[IS NOT NULL($84)]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), UserID], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"UserID":{"terms":{"field":"UserID","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"count()":"desc"},{"_key":"asc"}]},"aggregations":{"count()":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q17.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q17.yaml new file mode 100644 index 00000000000..b74403958e0 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q17.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(count()=[$2], UserID=[$0], SearchPhrase=[$1]) + LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) + LogicalProject(UserID=[$84], SearchPhrase=[$63]) + LogicalFilter(condition=[AND(IS NOT NULL($84), IS NOT NULL($63))]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[SearchPhrase, UserID], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[count(), UserID, SearchPhrase], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"UserID","boost":1.0}},{"exists":{"field":"SearchPhrase","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["SearchPhrase","UserID"],"excludes":[]},"aggregations":{"UserID|SearchPhrase":{"multi_terms":{"terms":[{"field":"UserID"},{"field":"SearchPhrase"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]},"aggregations":{"count()":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q18.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q18.yaml new file mode 100644 index 00000000000..c940061f690 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q18.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(fetch=[10]) + LogicalProject(count()=[$2], UserID=[$0], SearchPhrase=[$1]) + LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) + LogicalProject(UserID=[$84], SearchPhrase=[$63]) + LogicalFilter(condition=[AND(IS NOT NULL($84), IS NOT NULL($63))]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), UserID, SearchPhrase], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"SearchPhrase":{"terms":{"field":"SearchPhrase","missing_bucket":false,"order":"asc"}}},{"UserID":{"terms":{"field":"UserID","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q19.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q19.yaml new file mode 100644 index 00000000000..ccc50d9fd47 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q19.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(count()=[$3], UserID=[$0], m=[$1], SearchPhrase=[$2]) + LogicalAggregate(group=[{0, 1, 2}], count()=[COUNT()]) + LogicalProject(UserID=[$84], m=[$111], SearchPhrase=[$63]) + LogicalFilter(condition=[AND(IS NOT NULL($84), IS NOT NULL($111), IS NOT NULL($63))]) + LogicalProject(EventDate=[$0], URLRegionID=[$1], HasGCLID=[$2], Income=[$3], Interests=[$4], Robotness=[$5], BrowserLanguage=[$6], CounterClass=[$7], BrowserCountry=[$8], OriginalURL=[$9], ClientTimeZone=[$10], RefererHash=[$11], TraficSourceID=[$12], HitColor=[$13], RefererRegionID=[$14], URLCategoryID=[$15], LocalEventTime=[$16], EventTime=[$17], UTMTerm=[$18], AdvEngineID=[$19], UserAgentMinor=[$20], UserAgentMajor=[$21], RemoteIP=[$22], Sex=[$23], JavaEnable=[$24], URLHash=[$25], URL=[$26], ParamOrderID=[$27], OpenstatSourceID=[$28], HTTPError=[$29], SilverlightVersion3=[$30], MobilePhoneModel=[$31], SilverlightVersion4=[$32], SilverlightVersion1=[$33], SilverlightVersion2=[$34], IsDownload=[$35], IsParameter=[$36], CLID=[$37], FlashMajor=[$38], FlashMinor=[$39], UTMMedium=[$40], WatchID=[$41], DontCountHits=[$42], CookieEnable=[$43], HID=[$44], SocialAction=[$45], WindowName=[$46], ConnectTiming=[$47], PageCharset=[$48], IsLink=[$49], IsArtifical=[$50], JavascriptEnable=[$51], ClientEventTime=[$52], DNSTiming=[$53], CodeVersion=[$54], ResponseEndTiming=[$55], FUniqID=[$56], WindowClientHeight=[$57], OpenstatServiceName=[$58], UTMContent=[$59], HistoryLength=[$60], IsOldCounter=[$61], MobilePhone=[$62], SearchPhrase=[$63], FlashMinor2=[$64], SearchEngineID=[$65], IsEvent=[$66], UTMSource=[$67], RegionID=[$68], OpenstatAdID=[$69], UTMCampaign=[$70], GoodEvent=[$71], IsRefresh=[$72], ParamCurrency=[$73], Params=[$74], ResolutionHeight=[$75], ClientIP=[$76], FromTag=[$77], ParamCurrencyID=[$78], ResponseStartTiming=[$79], ResolutionWidth=[$80], SendTiming=[$81], RefererCategoryID=[$82], OpenstatCampaignID=[$83], UserID=[$84], WithHash=[$85], UserAgent=[$86], ParamPrice=[$87], ResolutionDepth=[$88], IsMobile=[$89], Age=[$90], SocialSourceNetworkID=[$91], OpenerName=[$92], OS=[$93], IsNotBounce=[$94], Referer=[$95], NetMinor=[$96], Title=[$97], NetMajor=[$98], IPNetworkID=[$99], FetchTiming=[$100], SocialNetwork=[$101], SocialSourcePage=[$102], CounterID=[$103], WindowClientWidth=[$104], _id=[$105], _index=[$106], _score=[$107], _maxscore=[$108], _sort=[$109], _routing=[$110], m=[EXTRACT('minute':VARCHAR, $17)]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableLimit(fetch=[10]) + EnumerableSort(sort0=[$0], dir0=[DESC-nulls-last]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2},count()=COUNT()), PROJECT->[count(), UserID, m, SearchPhrase]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"SearchPhrase":{"terms":{"field":"SearchPhrase","missing_bucket":false,"order":"asc"}}},{"UserID":{"terms":{"field":"UserID","missing_bucket":false,"order":"asc"}}},{"m":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAt3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiRXZlbnRUaW1lIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0Ae57CiAgIm9wIjogewogICAgIm5hbWUiOiAiRVhUUkFDVCIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAibGl0ZXJhbCI6ICJtaW51dGUiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQACUV2ZW50VGltZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGVUeXBlni1SrhB9yq8CAAFMAAdmb3JtYXRzdAAQTGphdmEvdXRpbC9MaXN0O3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzdAAPTGphdmEvdXRpbC9NYXA7eHB+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAlUSU1FU1RBTVB+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgASdAAERGF0ZXNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGXhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGwAAAABzcgATamF2YS51dGlsLkFycmF5TGlzdHiB0h2Zx2GdAwABSQAEc2l6ZXhwAAAAA3cEAAAAA3QAE3l5eXktTU0tZGQgSEg6bW06c3N0ABlzdHJpY3RfZGF0ZV9vcHRpb25hbF90aW1ldAAMZXBvY2hfbWlsbGlzeHh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":false,"value_type":"long","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q2.yaml new file mode 100644 index 00000000000..65149a1553b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q2.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], count()=[COUNT()]) + LogicalFilter(condition=[<>($19, 0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[AdvEngineID], FILTER-><>($0, 0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"AdvEngineID","boost":1.0}}],"must_not":[{"term":{"AdvEngineID":{"value":0,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["AdvEngineID"],"excludes":[]},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q20.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q20.yaml new file mode 100644 index 00000000000..0139a468f93 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q20.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(UserID=[$84]) + LogicalFilter(condition=[=($84, 435090932899640449)]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[UserID], FILTER->=($0, 435090932899640449), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"term":{"UserID":{"value":435090932899640449,"boost":1.0}}},"_source":{"includes":["UserID"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q21.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q21.yaml new file mode 100644 index 00000000000..88274ba5655 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q21.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], count()=[COUNT()]) + LogicalFilter(condition=[ILIKE($26, '%google%', '\')]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[URL], FILTER->ILIKE($0, '%google%', '\'), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"wildcard":{"URL":{"wildcard":"*google*","case_insensitive":true,"boost":1.0}}},"_source":{"includes":["URL"],"excludes":[]},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q22.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q22.yaml new file mode 100644 index 00000000000..1fec253178c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q22.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(c=[$1], SearchPhrase=[$0]) + LogicalAggregate(group=[{0}], c=[COUNT()]) + LogicalProject(SearchPhrase=[$63]) + LogicalFilter(condition=[IS NOT NULL($63)]) + LogicalFilter(condition=[AND(ILIKE($26, '%google%', '\'), <>($63, ''))]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[URL, SearchPhrase], FILTER->AND(ILIKE($0, '%google%', '\'), <>($1, '')), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[c, SearchPhrase], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"wildcard":{"URL":{"wildcard":"*google*","case_insensitive":true,"boost":1.0}}},{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["URL","SearchPhrase"],"excludes":[]},"aggregations":{"SearchPhrase":{"terms":{"field":"SearchPhrase","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q23.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q23.yaml new file mode 100644 index 00000000000..f258552964f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q23.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(c=[$1], dc(UserID)=[$2], SearchPhrase=[$0]) + LogicalAggregate(group=[{0}], c=[COUNT()], dc(UserID)=[COUNT(DISTINCT $1)]) + LogicalProject(SearchPhrase=[$63], UserID=[$84]) + LogicalFilter(condition=[IS NOT NULL($63)]) + LogicalFilter(condition=[AND(ILIKE($97, '%Google%', '\'), <>($63, ''), NOT(ILIKE($26, '%.google.%', '\')))]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableLimit(fetch=[10]) + EnumerableSort(sort0=[$0], dir0=[DESC-nulls-last]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[URL, SearchPhrase, UserID, Title], FILTER->AND(ILIKE($3, '%Google%', '\'), <>($1, ''), NOT(ILIKE($0, '%.google.%', '\'))), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT(),dc(UserID)=COUNT(DISTINCT $1)), PROJECT->[c, dc(UserID), SearchPhrase]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"wildcard":{"Title":{"wildcard":"*Google*","case_insensitive":true,"boost":1.0}}},{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"bool":{"must_not":[{"wildcard":{"URL":{"wildcard":"*.google.*","case_insensitive":true,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["URL","SearchPhrase","UserID","Title"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"SearchPhrase":{"terms":{"field":"SearchPhrase","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"dc(UserID)":{"cardinality":{"field":"UserID"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q24.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q24.yaml new file mode 100644 index 00000000000..97c0970f8d6 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q24.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(EventDate=[$0], URLRegionID=[$1], HasGCLID=[$2], Income=[$3], Interests=[$4], Robotness=[$5], BrowserLanguage=[$6], CounterClass=[$7], BrowserCountry=[$8], OriginalURL=[$9], ClientTimeZone=[$10], RefererHash=[$11], TraficSourceID=[$12], HitColor=[$13], RefererRegionID=[$14], URLCategoryID=[$15], LocalEventTime=[$16], EventTime=[$17], UTMTerm=[$18], AdvEngineID=[$19], UserAgentMinor=[$20], UserAgentMajor=[$21], RemoteIP=[$22], Sex=[$23], JavaEnable=[$24], URLHash=[$25], URL=[$26], ParamOrderID=[$27], OpenstatSourceID=[$28], HTTPError=[$29], SilverlightVersion3=[$30], MobilePhoneModel=[$31], SilverlightVersion4=[$32], SilverlightVersion1=[$33], SilverlightVersion2=[$34], IsDownload=[$35], IsParameter=[$36], CLID=[$37], FlashMajor=[$38], FlashMinor=[$39], UTMMedium=[$40], WatchID=[$41], DontCountHits=[$42], CookieEnable=[$43], HID=[$44], SocialAction=[$45], WindowName=[$46], ConnectTiming=[$47], PageCharset=[$48], IsLink=[$49], IsArtifical=[$50], JavascriptEnable=[$51], ClientEventTime=[$52], DNSTiming=[$53], CodeVersion=[$54], ResponseEndTiming=[$55], FUniqID=[$56], WindowClientHeight=[$57], OpenstatServiceName=[$58], UTMContent=[$59], HistoryLength=[$60], IsOldCounter=[$61], MobilePhone=[$62], SearchPhrase=[$63], FlashMinor2=[$64], SearchEngineID=[$65], IsEvent=[$66], UTMSource=[$67], RegionID=[$68], OpenstatAdID=[$69], UTMCampaign=[$70], GoodEvent=[$71], IsRefresh=[$72], ParamCurrency=[$73], Params=[$74], ResolutionHeight=[$75], ClientIP=[$76], FromTag=[$77], ParamCurrencyID=[$78], ResponseStartTiming=[$79], ResolutionWidth=[$80], SendTiming=[$81], RefererCategoryID=[$82], OpenstatCampaignID=[$83], UserID=[$84], WithHash=[$85], UserAgent=[$86], ParamPrice=[$87], ResolutionDepth=[$88], IsMobile=[$89], Age=[$90], SocialSourceNetworkID=[$91], OpenerName=[$92], OS=[$93], IsNotBounce=[$94], Referer=[$95], NetMinor=[$96], Title=[$97], NetMajor=[$98], IPNetworkID=[$99], FetchTiming=[$100], SocialNetwork=[$101], SocialSourcePage=[$102], CounterID=[$103], WindowClientWidth=[$104]) + LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) + LogicalFilter(condition=[ILIKE($26, '%google%', '\')]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[EventDate, URLRegionID, HasGCLID, Income, Interests, Robotness, BrowserLanguage, CounterClass, BrowserCountry, OriginalURL, ClientTimeZone, RefererHash, TraficSourceID, HitColor, RefererRegionID, URLCategoryID, LocalEventTime, EventTime, UTMTerm, AdvEngineID, UserAgentMinor, UserAgentMajor, RemoteIP, Sex, JavaEnable, URLHash, URL, ParamOrderID, OpenstatSourceID, HTTPError, SilverlightVersion3, MobilePhoneModel, SilverlightVersion4, SilverlightVersion1, SilverlightVersion2, IsDownload, IsParameter, CLID, FlashMajor, FlashMinor, UTMMedium, WatchID, DontCountHits, CookieEnable, HID, SocialAction, WindowName, ConnectTiming, PageCharset, IsLink, IsArtifical, JavascriptEnable, ClientEventTime, DNSTiming, CodeVersion, ResponseEndTiming, FUniqID, WindowClientHeight, OpenstatServiceName, UTMContent, HistoryLength, IsOldCounter, MobilePhone, SearchPhrase, FlashMinor2, SearchEngineID, IsEvent, UTMSource, RegionID, OpenstatAdID, UTMCampaign, GoodEvent, IsRefresh, ParamCurrency, Params, ResolutionHeight, ClientIP, FromTag, ParamCurrencyID, ResponseStartTiming, ResolutionWidth, SendTiming, RefererCategoryID, OpenstatCampaignID, UserID, WithHash, UserAgent, ParamPrice, ResolutionDepth, IsMobile, Age, SocialSourceNetworkID, OpenerName, OS, IsNotBounce, Referer, NetMinor, Title, NetMajor, IPNetworkID, FetchTiming, SocialNetwork, SocialSourcePage, CounterID, WindowClientWidth], FILTER->ILIKE($26, '%google%', '\'), SORT->[{ + "EventTime" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"wildcard":{"URL":{"wildcard":"*google*","case_insensitive":true,"boost":1.0}}},"_source":{"includes":["EventDate","URLRegionID","HasGCLID","Income","Interests","Robotness","BrowserLanguage","CounterClass","BrowserCountry","OriginalURL","ClientTimeZone","RefererHash","TraficSourceID","HitColor","RefererRegionID","URLCategoryID","LocalEventTime","EventTime","UTMTerm","AdvEngineID","UserAgentMinor","UserAgentMajor","RemoteIP","Sex","JavaEnable","URLHash","URL","ParamOrderID","OpenstatSourceID","HTTPError","SilverlightVersion3","MobilePhoneModel","SilverlightVersion4","SilverlightVersion1","SilverlightVersion2","IsDownload","IsParameter","CLID","FlashMajor","FlashMinor","UTMMedium","WatchID","DontCountHits","CookieEnable","HID","SocialAction","WindowName","ConnectTiming","PageCharset","IsLink","IsArtifical","JavascriptEnable","ClientEventTime","DNSTiming","CodeVersion","ResponseEndTiming","FUniqID","WindowClientHeight","OpenstatServiceName","UTMContent","HistoryLength","IsOldCounter","MobilePhone","SearchPhrase","FlashMinor2","SearchEngineID","IsEvent","UTMSource","RegionID","OpenstatAdID","UTMCampaign","GoodEvent","IsRefresh","ParamCurrency","Params","ResolutionHeight","ClientIP","FromTag","ParamCurrencyID","ResponseStartTiming","ResolutionWidth","SendTiming","RefererCategoryID","OpenstatCampaignID","UserID","WithHash","UserAgent","ParamPrice","ResolutionDepth","IsMobile","Age","SocialSourceNetworkID","OpenerName","OS","IsNotBounce","Referer","NetMinor","Title","NetMajor","IPNetworkID","FetchTiming","SocialNetwork","SocialSourcePage","CounterID","WindowClientWidth"],"excludes":[]},"sort":[{"EventTime":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q25.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q25.yaml new file mode 100644 index 00000000000..612b8bc06f8 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q25.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(SearchPhrase=[$63]) + LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) + LogicalFilter(condition=[<>($63, '')]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[EventTime, SearchPhrase], FILTER-><>($1, ''), SORT->[{ + "EventTime" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, PROJECT->[SearchPhrase], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["SearchPhrase"],"excludes":[]},"sort":[{"EventTime":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q26.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q26.yaml new file mode 100644 index 00000000000..233a58c19c6 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q26.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10]) + LogicalProject(SearchPhrase=[$63]) + LogicalFilter(condition=[<>($63, '')]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[SearchPhrase], FILTER-><>($0, ''), SORT->[{ + "SearchPhrase" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["SearchPhrase"],"excludes":[]},"sort":[{"SearchPhrase":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q27.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q27.yaml new file mode 100644 index 00000000000..1da73eb16c8 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q27.yaml @@ -0,0 +1,19 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(SearchPhrase=[$63]) + LogicalSort(sort0=[$17], sort1=[$63], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10]) + LogicalFilter(condition=[<>($63, '')]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[EventTime, SearchPhrase], FILTER-><>($1, ''), SORT->[{ + "EventTime" : { + "order" : "asc", + "missing" : "_first" + } + }, { + "SearchPhrase" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, PROJECT->[SearchPhrase], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["SearchPhrase"],"excludes":[]},"sort":[{"EventTime":{"order":"asc","missing":"_first"}},{"SearchPhrase":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q28.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q28.yaml new file mode 100644 index 00000000000..838a201cf92 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q28.yaml @@ -0,0 +1,17 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[25]) + LogicalFilter(condition=[>($1, 100000)]) + LogicalProject(l=[$1], c=[$2], CounterID=[$0]) + LogicalAggregate(group=[{0}], l=[AVG($1)], c=[COUNT()]) + LogicalProject(CounterID=[$103], $f2=[CHAR_LENGTH($26)]) + LogicalFilter(condition=[IS NOT NULL($103)]) + LogicalFilter(condition=[<>($26, '')]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableLimit(fetch=[25]) + EnumerableSort(sort0=[$0], dir0=[DESC-nulls-last]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[100000], expr#4=[>($t1, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[URL, CounterID], FILTER->AND(<>($0, ''), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},l=AVG($1),c=COUNT()), PROJECT->[l, c, CounterID]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"URL","boost":1.0}}],"must_not":[{"term":{"URL":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"CounterID","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["URL","CounterID"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"CounterID":{"terms":{"field":"CounterID","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"l":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAknsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJVUkwiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQApnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiAwLAogICAgICAibmFtZSI6ICIkMCIKICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANVUkx+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAZTVFJJTkd4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q29.yaml similarity index 100% rename from integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.json rename to integ-test/src/test/resources/expectedOutput/calcite/clickbench/q29.yaml diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q3.yaml new file mode 100644 index 00000000000..ef93b63ee80 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q3.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], sum(AdvEngineID)=[SUM($0)], count()=[COUNT()], avg(ResolutionWidth)=[AVG($1)]) + LogicalProject(AdvEngineID=[$19], ResolutionWidth=[$80]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},sum(AdvEngineID)=SUM($0),count()=COUNT(),avg(ResolutionWidth)=AVG($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"sum(AdvEngineID)":{"sum":{"field":"AdvEngineID"}},"count()":{"value_count":{"field":"_index"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml new file mode 100644 index 00000000000..de1f6d31f0d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], sum(ResolutionWidth)=[SUM($0)], sum(ResolutionWidth+1)=[SUM($1)], sum(ResolutionWidth+2)=[SUM($2)], sum(ResolutionWidth+3)=[SUM($3)], sum(ResolutionWidth+4)=[SUM($4)], sum(ResolutionWidth+5)=[SUM($5)], sum(ResolutionWidth+6)=[SUM($6)], sum(ResolutionWidth+7)=[SUM($7)], sum(ResolutionWidth+8)=[SUM($8)], sum(ResolutionWidth+9)=[SUM($9)], sum(ResolutionWidth+10)=[SUM($10)], sum(ResolutionWidth+11)=[SUM($11)], sum(ResolutionWidth+12)=[SUM($12)], sum(ResolutionWidth+13)=[SUM($13)], sum(ResolutionWidth+14)=[SUM($14)], sum(ResolutionWidth+15)=[SUM($15)], sum(ResolutionWidth+16)=[SUM($16)], sum(ResolutionWidth+17)=[SUM($17)], sum(ResolutionWidth+18)=[SUM($18)], sum(ResolutionWidth+19)=[SUM($19)], sum(ResolutionWidth+20)=[SUM($20)], sum(ResolutionWidth+21)=[SUM($21)], sum(ResolutionWidth+22)=[SUM($22)], sum(ResolutionWidth+23)=[SUM($23)], sum(ResolutionWidth+24)=[SUM($24)], sum(ResolutionWidth+25)=[SUM($25)], sum(ResolutionWidth+26)=[SUM($26)], sum(ResolutionWidth+27)=[SUM($27)], sum(ResolutionWidth+28)=[SUM($28)], sum(ResolutionWidth+29)=[SUM($29)], sum(ResolutionWidth+30)=[SUM($30)], sum(ResolutionWidth+31)=[SUM($31)], sum(ResolutionWidth+32)=[SUM($32)], sum(ResolutionWidth+33)=[SUM($33)], sum(ResolutionWidth+34)=[SUM($34)], sum(ResolutionWidth+35)=[SUM($35)], sum(ResolutionWidth+36)=[SUM($36)], sum(ResolutionWidth+37)=[SUM($37)], sum(ResolutionWidth+38)=[SUM($38)], sum(ResolutionWidth+39)=[SUM($39)], sum(ResolutionWidth+40)=[SUM($40)], sum(ResolutionWidth+41)=[SUM($41)], sum(ResolutionWidth+42)=[SUM($42)], sum(ResolutionWidth+43)=[SUM($43)], sum(ResolutionWidth+44)=[SUM($44)], sum(ResolutionWidth+45)=[SUM($45)], sum(ResolutionWidth+46)=[SUM($46)], sum(ResolutionWidth+47)=[SUM($47)], sum(ResolutionWidth+48)=[SUM($48)], sum(ResolutionWidth+49)=[SUM($49)], sum(ResolutionWidth+50)=[SUM($50)], sum(ResolutionWidth+51)=[SUM($51)], sum(ResolutionWidth+52)=[SUM($52)], sum(ResolutionWidth+53)=[SUM($53)], sum(ResolutionWidth+54)=[SUM($54)], sum(ResolutionWidth+55)=[SUM($55)], sum(ResolutionWidth+56)=[SUM($56)], sum(ResolutionWidth+57)=[SUM($57)], sum(ResolutionWidth+58)=[SUM($58)], sum(ResolutionWidth+59)=[SUM($59)], sum(ResolutionWidth+60)=[SUM($60)], sum(ResolutionWidth+61)=[SUM($61)], sum(ResolutionWidth+62)=[SUM($62)], sum(ResolutionWidth+63)=[SUM($63)], sum(ResolutionWidth+64)=[SUM($64)], sum(ResolutionWidth+65)=[SUM($65)], sum(ResolutionWidth+66)=[SUM($66)], sum(ResolutionWidth+67)=[SUM($67)], sum(ResolutionWidth+68)=[SUM($68)], sum(ResolutionWidth+69)=[SUM($69)], sum(ResolutionWidth+70)=[SUM($70)], sum(ResolutionWidth+71)=[SUM($71)], sum(ResolutionWidth+72)=[SUM($72)], sum(ResolutionWidth+73)=[SUM($73)], sum(ResolutionWidth+74)=[SUM($74)], sum(ResolutionWidth+75)=[SUM($75)], sum(ResolutionWidth+76)=[SUM($76)], sum(ResolutionWidth+77)=[SUM($77)], sum(ResolutionWidth+78)=[SUM($78)], sum(ResolutionWidth+79)=[SUM($79)], sum(ResolutionWidth+80)=[SUM($80)], sum(ResolutionWidth+81)=[SUM($81)], sum(ResolutionWidth+82)=[SUM($82)], sum(ResolutionWidth+83)=[SUM($83)], sum(ResolutionWidth+84)=[SUM($84)], sum(ResolutionWidth+85)=[SUM($85)], sum(ResolutionWidth+86)=[SUM($86)], sum(ResolutionWidth+87)=[SUM($87)], sum(ResolutionWidth+88)=[SUM($88)], sum(ResolutionWidth+89)=[SUM($89)]) + LogicalProject(ResolutionWidth=[$80], $f90=[+($80, 1)], $f91=[+($80, 2)], $f92=[+($80, 3)], $f93=[+($80, 4)], $f94=[+($80, 5)], $f95=[+($80, 6)], $f96=[+($80, 7)], $f97=[+($80, 8)], $f98=[+($80, 9)], $f99=[+($80, 10)], $f100=[+($80, 11)], $f101=[+($80, 12)], $f102=[+($80, 13)], $f103=[+($80, 14)], $f104=[+($80, 15)], $f105=[+($80, 16)], $f106=[+($80, 17)], $f107=[+($80, 18)], $f108=[+($80, 19)], $f109=[+($80, 20)], $f110=[+($80, 21)], $f111=[+($80, 22)], $f112=[+($80, 23)], $f113=[+($80, 24)], $f114=[+($80, 25)], $f115=[+($80, 26)], $f116=[+($80, 27)], $f117=[+($80, 28)], $f118=[+($80, 29)], $f119=[+($80, 30)], $f120=[+($80, 31)], $f121=[+($80, 32)], $f122=[+($80, 33)], $f123=[+($80, 34)], $f124=[+($80, 35)], $f125=[+($80, 36)], $f126=[+($80, 37)], $f127=[+($80, 38)], $f128=[+($80, 39)], $f129=[+($80, 40)], $f130=[+($80, 41)], $f131=[+($80, 42)], $f132=[+($80, 43)], $f133=[+($80, 44)], $f134=[+($80, 45)], $f135=[+($80, 46)], $f136=[+($80, 47)], $f137=[+($80, 48)], $f138=[+($80, 49)], $f139=[+($80, 50)], $f140=[+($80, 51)], $f141=[+($80, 52)], $f142=[+($80, 53)], $f143=[+($80, 54)], $f144=[+($80, 55)], $f145=[+($80, 56)], $f146=[+($80, 57)], $f147=[+($80, 58)], $f148=[+($80, 59)], $f149=[+($80, 60)], $f150=[+($80, 61)], $f151=[+($80, 62)], $f152=[+($80, 63)], $f153=[+($80, 64)], $f154=[+($80, 65)], $f155=[+($80, 66)], $f156=[+($80, 67)], $f157=[+($80, 68)], $f158=[+($80, 69)], $f159=[+($80, 70)], $f160=[+($80, 71)], $f161=[+($80, 72)], $f162=[+($80, 73)], $f163=[+($80, 74)], $f164=[+($80, 75)], $f165=[+($80, 76)], $f166=[+($80, 77)], $f167=[+($80, 78)], $f168=[+($80, 79)], $f169=[+($80, 80)], $f170=[+($80, 81)], $f171=[+($80, 82)], $f172=[+($80, 83)], $f173=[+($80, 84)], $f174=[+($80, 85)], $f175=[+($80, 86)], $f176=[+($80, 87)], $f177=[+($80, 88)], $f178=[+($80, 89)]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t1):BIGINT], expr#3=[+($t0, $t2)], expr#4=[2], expr#5=[*($t1, $t4)], expr#6=[+($t0, $t5)], expr#7=[3], expr#8=[*($t1, $t7)], expr#9=[+($t0, $t8)], expr#10=[4], expr#11=[*($t1, $t10)], expr#12=[+($t0, $t11)], expr#13=[5], expr#14=[*($t1, $t13)], expr#15=[+($t0, $t14)], expr#16=[6], expr#17=[*($t1, $t16)], expr#18=[+($t0, $t17)], expr#19=[7], expr#20=[*($t1, $t19)], expr#21=[+($t0, $t20)], expr#22=[8], expr#23=[*($t1, $t22)], expr#24=[+($t0, $t23)], expr#25=[9], expr#26=[*($t1, $t25)], expr#27=[+($t0, $t26)], expr#28=[10], expr#29=[*($t1, $t28)], expr#30=[+($t0, $t29)], expr#31=[11], expr#32=[*($t1, $t31)], expr#33=[+($t0, $t32)], expr#34=[12], expr#35=[*($t1, $t34)], expr#36=[+($t0, $t35)], expr#37=[13], expr#38=[*($t1, $t37)], expr#39=[+($t0, $t38)], expr#40=[14], expr#41=[*($t1, $t40)], expr#42=[+($t0, $t41)], expr#43=[15], expr#44=[*($t1, $t43)], expr#45=[+($t0, $t44)], expr#46=[16], expr#47=[*($t1, $t46)], expr#48=[+($t0, $t47)], expr#49=[17], expr#50=[*($t1, $t49)], expr#51=[+($t0, $t50)], expr#52=[18], expr#53=[*($t1, $t52)], expr#54=[+($t0, $t53)], expr#55=[19], expr#56=[*($t1, $t55)], expr#57=[+($t0, $t56)], expr#58=[20], expr#59=[*($t1, $t58)], expr#60=[+($t0, $t59)], expr#61=[21], expr#62=[*($t1, $t61)], expr#63=[+($t0, $t62)], expr#64=[22], expr#65=[*($t1, $t64)], expr#66=[+($t0, $t65)], expr#67=[23], expr#68=[*($t1, $t67)], expr#69=[+($t0, $t68)], expr#70=[24], expr#71=[*($t1, $t70)], expr#72=[+($t0, $t71)], expr#73=[25], expr#74=[*($t1, $t73)], expr#75=[+($t0, $t74)], expr#76=[26], expr#77=[*($t1, $t76)], expr#78=[+($t0, $t77)], expr#79=[27], expr#80=[*($t1, $t79)], expr#81=[+($t0, $t80)], expr#82=[28], expr#83=[*($t1, $t82)], expr#84=[+($t0, $t83)], expr#85=[29], expr#86=[*($t1, $t85)], expr#87=[+($t0, $t86)], expr#88=[30], expr#89=[*($t1, $t88)], expr#90=[+($t0, $t89)], expr#91=[31], expr#92=[*($t1, $t91)], expr#93=[+($t0, $t92)], expr#94=[32], expr#95=[*($t1, $t94)], expr#96=[+($t0, $t95)], expr#97=[33], expr#98=[*($t1, $t97)], expr#99=[+($t0, $t98)], expr#100=[34], expr#101=[*($t1, $t100)], expr#102=[+($t0, $t101)], expr#103=[35], expr#104=[*($t1, $t103)], expr#105=[+($t0, $t104)], expr#106=[36], expr#107=[*($t1, $t106)], expr#108=[+($t0, $t107)], expr#109=[37], expr#110=[*($t1, $t109)], expr#111=[+($t0, $t110)], expr#112=[38], expr#113=[*($t1, $t112)], expr#114=[+($t0, $t113)], expr#115=[39], expr#116=[*($t1, $t115)], expr#117=[+($t0, $t116)], expr#118=[40], expr#119=[*($t1, $t118)], expr#120=[+($t0, $t119)], expr#121=[41], expr#122=[*($t1, $t121)], expr#123=[+($t0, $t122)], expr#124=[42], expr#125=[*($t1, $t124)], expr#126=[+($t0, $t125)], expr#127=[43], expr#128=[*($t1, $t127)], expr#129=[+($t0, $t128)], expr#130=[44], expr#131=[*($t1, $t130)], expr#132=[+($t0, $t131)], expr#133=[45], expr#134=[*($t1, $t133)], expr#135=[+($t0, $t134)], expr#136=[46], expr#137=[*($t1, $t136)], expr#138=[+($t0, $t137)], expr#139=[47], expr#140=[*($t1, $t139)], expr#141=[+($t0, $t140)], expr#142=[48], expr#143=[*($t1, $t142)], expr#144=[+($t0, $t143)], expr#145=[49], expr#146=[*($t1, $t145)], expr#147=[+($t0, $t146)], expr#148=[50], expr#149=[*($t1, $t148)], expr#150=[+($t0, $t149)], expr#151=[51], expr#152=[*($t1, $t151)], expr#153=[+($t0, $t152)], expr#154=[52], expr#155=[*($t1, $t154)], expr#156=[+($t0, $t155)], expr#157=[53], expr#158=[*($t1, $t157)], expr#159=[+($t0, $t158)], expr#160=[54], expr#161=[*($t1, $t160)], expr#162=[+($t0, $t161)], expr#163=[55], expr#164=[*($t1, $t163)], expr#165=[+($t0, $t164)], expr#166=[56], expr#167=[*($t1, $t166)], expr#168=[+($t0, $t167)], expr#169=[57], expr#170=[*($t1, $t169)], expr#171=[+($t0, $t170)], expr#172=[58], expr#173=[*($t1, $t172)], expr#174=[+($t0, $t173)], expr#175=[59], expr#176=[*($t1, $t175)], expr#177=[+($t0, $t176)], expr#178=[60], expr#179=[*($t1, $t178)], expr#180=[+($t0, $t179)], expr#181=[61], expr#182=[*($t1, $t181)], expr#183=[+($t0, $t182)], expr#184=[62], expr#185=[*($t1, $t184)], expr#186=[+($t0, $t185)], expr#187=[63], expr#188=[*($t1, $t187)], expr#189=[+($t0, $t188)], expr#190=[64], expr#191=[*($t1, $t190)], expr#192=[+($t0, $t191)], expr#193=[65], expr#194=[*($t1, $t193)], expr#195=[+($t0, $t194)], expr#196=[66], expr#197=[*($t1, $t196)], expr#198=[+($t0, $t197)], expr#199=[67], expr#200=[*($t1, $t199)], expr#201=[+($t0, $t200)], expr#202=[68], expr#203=[*($t1, $t202)], expr#204=[+($t0, $t203)], expr#205=[69], expr#206=[*($t1, $t205)], expr#207=[+($t0, $t206)], expr#208=[70], expr#209=[*($t1, $t208)], expr#210=[+($t0, $t209)], expr#211=[71], expr#212=[*($t1, $t211)], expr#213=[+($t0, $t212)], expr#214=[72], expr#215=[*($t1, $t214)], expr#216=[+($t0, $t215)], expr#217=[73], expr#218=[*($t1, $t217)], expr#219=[+($t0, $t218)], expr#220=[74], expr#221=[*($t1, $t220)], expr#222=[+($t0, $t221)], expr#223=[75], expr#224=[*($t1, $t223)], expr#225=[+($t0, $t224)], expr#226=[76], expr#227=[*($t1, $t226)], expr#228=[+($t0, $t227)], expr#229=[77], expr#230=[*($t1, $t229)], expr#231=[+($t0, $t230)], expr#232=[78], expr#233=[*($t1, $t232)], expr#234=[+($t0, $t233)], expr#235=[79], expr#236=[*($t1, $t235)], expr#237=[+($t0, $t236)], expr#238=[80], expr#239=[*($t1, $t238)], expr#240=[+($t0, $t239)], expr#241=[81], expr#242=[*($t1, $t241)], expr#243=[+($t0, $t242)], expr#244=[82], expr#245=[*($t1, $t244)], expr#246=[+($t0, $t245)], expr#247=[83], expr#248=[*($t1, $t247)], expr#249=[+($t0, $t248)], expr#250=[84], expr#251=[*($t1, $t250)], expr#252=[+($t0, $t251)], expr#253=[85], expr#254=[*($t1, $t253)], expr#255=[+($t0, $t254)], expr#256=[86], expr#257=[*($t1, $t256)], expr#258=[+($t0, $t257)], expr#259=[87], expr#260=[*($t1, $t259)], expr#261=[+($t0, $t260)], expr#262=[88], expr#263=[*($t1, $t262)], expr#264=[+($t0, $t263)], expr#265=[89], expr#266=[*($t1, $t265)], expr#267=[+($t0, $t266)], sum(ResolutionWidth)=[$t0], sum(ResolutionWidth+1)=[$t3], sum(ResolutionWidth+2)=[$t6], sum(ResolutionWidth+3)=[$t9], sum(ResolutionWidth+4)=[$t12], sum(ResolutionWidth+5)=[$t15], sum(ResolutionWidth+6)=[$t18], sum(ResolutionWidth+7)=[$t21], sum(ResolutionWidth+8)=[$t24], sum(ResolutionWidth+9)=[$t27], sum(ResolutionWidth+10)=[$t30], sum(ResolutionWidth+11)=[$t33], sum(ResolutionWidth+12)=[$t36], sum(ResolutionWidth+13)=[$t39], sum(ResolutionWidth+14)=[$t42], sum(ResolutionWidth+15)=[$t45], sum(ResolutionWidth+16)=[$t48], sum(ResolutionWidth+17)=[$t51], sum(ResolutionWidth+18)=[$t54], sum(ResolutionWidth+19)=[$t57], sum(ResolutionWidth+20)=[$t60], sum(ResolutionWidth+21)=[$t63], sum(ResolutionWidth+22)=[$t66], sum(ResolutionWidth+23)=[$t69], sum(ResolutionWidth+24)=[$t72], sum(ResolutionWidth+25)=[$t75], sum(ResolutionWidth+26)=[$t78], sum(ResolutionWidth+27)=[$t81], sum(ResolutionWidth+28)=[$t84], sum(ResolutionWidth+29)=[$t87], sum(ResolutionWidth+30)=[$t90], sum(ResolutionWidth+31)=[$t93], sum(ResolutionWidth+32)=[$t96], sum(ResolutionWidth+33)=[$t99], sum(ResolutionWidth+34)=[$t102], sum(ResolutionWidth+35)=[$t105], sum(ResolutionWidth+36)=[$t108], sum(ResolutionWidth+37)=[$t111], sum(ResolutionWidth+38)=[$t114], sum(ResolutionWidth+39)=[$t117], sum(ResolutionWidth+40)=[$t120], sum(ResolutionWidth+41)=[$t123], sum(ResolutionWidth+42)=[$t126], sum(ResolutionWidth+43)=[$t129], sum(ResolutionWidth+44)=[$t132], sum(ResolutionWidth+45)=[$t135], sum(ResolutionWidth+46)=[$t138], sum(ResolutionWidth+47)=[$t141], sum(ResolutionWidth+48)=[$t144], sum(ResolutionWidth+49)=[$t147], sum(ResolutionWidth+50)=[$t150], sum(ResolutionWidth+51)=[$t153], sum(ResolutionWidth+52)=[$t156], sum(ResolutionWidth+53)=[$t159], sum(ResolutionWidth+54)=[$t162], sum(ResolutionWidth+55)=[$t165], sum(ResolutionWidth+56)=[$t168], sum(ResolutionWidth+57)=[$t171], sum(ResolutionWidth+58)=[$t174], sum(ResolutionWidth+59)=[$t177], sum(ResolutionWidth+60)=[$t180], sum(ResolutionWidth+61)=[$t183], sum(ResolutionWidth+62)=[$t186], sum(ResolutionWidth+63)=[$t189], sum(ResolutionWidth+64)=[$t192], sum(ResolutionWidth+65)=[$t195], sum(ResolutionWidth+66)=[$t198], sum(ResolutionWidth+67)=[$t201], sum(ResolutionWidth+68)=[$t204], sum(ResolutionWidth+69)=[$t207], sum(ResolutionWidth+70)=[$t210], sum(ResolutionWidth+71)=[$t213], sum(ResolutionWidth+72)=[$t216], sum(ResolutionWidth+73)=[$t219], sum(ResolutionWidth+74)=[$t222], sum(ResolutionWidth+75)=[$t225], sum(ResolutionWidth+76)=[$t228], sum(ResolutionWidth+77)=[$t231], sum(ResolutionWidth+78)=[$t234], sum(ResolutionWidth+79)=[$t237], sum(ResolutionWidth+80)=[$t240], sum(ResolutionWidth+81)=[$t243], sum(ResolutionWidth+82)=[$t246], sum(ResolutionWidth+83)=[$t249], sum(ResolutionWidth+84)=[$t252], sum(ResolutionWidth+85)=[$t255], sum(ResolutionWidth+86)=[$t258], sum(ResolutionWidth+87)=[$t261], sum(ResolutionWidth+88)=[$t264], sum(ResolutionWidth+89)=[$t267]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},sum(ResolutionWidth)=SUM($0),sum(ResolutionWidth+1)_COUNT=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"sum(ResolutionWidth)":{"sum":{"field":"ResolutionWidth"}},"sum(ResolutionWidth+1)_COUNT":{"value_count":{"field":"ResolutionWidth"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q31.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q31.yaml new file mode 100644 index 00000000000..a0bab4f2aed --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q31.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(c=[$2], sum(IsRefresh)=[$3], avg(ResolutionWidth)=[$4], SearchEngineID=[$0], ClientIP=[$1]) + LogicalAggregate(group=[{0, 1}], c=[COUNT()], sum(IsRefresh)=[SUM($2)], avg(ResolutionWidth)=[AVG($3)]) + LogicalProject(SearchEngineID=[$65], ClientIP=[$76], IsRefresh=[$72], ResolutionWidth=[$80]) + LogicalFilter(condition=[AND(IS NOT NULL($65), IS NOT NULL($76))]) + LogicalFilter(condition=[<>($63, '')]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableLimit(fetch=[10]) + EnumerableSort(sort0=[$0], dir0=[DESC-nulls-last]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[SearchPhrase, SearchEngineID, IsRefresh, ClientIP, ResolutionWidth], FILTER->AND(<>($0, ''), IS NOT NULL($1), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},c=COUNT(),sum(IsRefresh)=SUM($2),avg(ResolutionWidth)=AVG($3)), PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), SearchEngineID, ClientIP]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"SearchEngineID","boost":1.0}},{"exists":{"field":"ClientIP","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["SearchPhrase","SearchEngineID","IsRefresh","ClientIP","ResolutionWidth"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"SearchEngineID":{"terms":{"field":"SearchEngineID","missing_bucket":false,"order":"asc"}}},{"ClientIP":{"terms":{"field":"ClientIP","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q32.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q32.yaml new file mode 100644 index 00000000000..60e5f7af061 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q32.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(c=[$2], sum(IsRefresh)=[$3], avg(ResolutionWidth)=[$4], WatchID=[$0], ClientIP=[$1]) + LogicalAggregate(group=[{0, 1}], c=[COUNT()], sum(IsRefresh)=[SUM($2)], avg(ResolutionWidth)=[AVG($3)]) + LogicalProject(WatchID=[$41], ClientIP=[$76], IsRefresh=[$72], ResolutionWidth=[$80]) + LogicalFilter(condition=[AND(IS NOT NULL($41), IS NOT NULL($76))]) + LogicalFilter(condition=[<>($63, '')]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableLimit(fetch=[10]) + EnumerableSort(sort0=[$0], dir0=[DESC-nulls-last]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[WatchID, SearchPhrase, IsRefresh, ClientIP, ResolutionWidth], FILTER->AND(<>($1, ''), IS NOT NULL($0), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},c=COUNT(),sum(IsRefresh)=SUM($2),avg(ResolutionWidth)=AVG($3)), PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), WatchID, ClientIP]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"WatchID","boost":1.0}},{"exists":{"field":"ClientIP","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["WatchID","SearchPhrase","IsRefresh","ClientIP","ResolutionWidth"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"WatchID":{"terms":{"field":"WatchID","missing_bucket":false,"order":"asc"}}},{"ClientIP":{"terms":{"field":"ClientIP","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q33.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q33.yaml new file mode 100644 index 00000000000..998d052f16e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q33.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(c=[$2], sum(IsRefresh)=[$3], avg(ResolutionWidth)=[$4], WatchID=[$0], ClientIP=[$1]) + LogicalAggregate(group=[{0, 1}], c=[COUNT()], sum(IsRefresh)=[SUM($2)], avg(ResolutionWidth)=[AVG($3)]) + LogicalProject(WatchID=[$41], ClientIP=[$76], IsRefresh=[$72], ResolutionWidth=[$80]) + LogicalFilter(condition=[AND(IS NOT NULL($41), IS NOT NULL($76))]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableLimit(fetch=[10]) + EnumerableSort(sort0=[$0], dir0=[DESC-nulls-last]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},c=COUNT(),sum(IsRefresh)=SUM($1),avg(ResolutionWidth)=AVG($3)), PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), WatchID, ClientIP]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"WatchID":{"terms":{"field":"WatchID","missing_bucket":false,"order":"asc"}}},{"ClientIP":{"terms":{"field":"ClientIP","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q34.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q34.yaml new file mode 100644 index 00000000000..220e94f3bbb --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q34.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(c=[$1], URL=[$0]) + LogicalAggregate(group=[{0}], c=[COUNT()]) + LogicalProject(URL=[$26]) + LogicalFilter(condition=[IS NOT NULL($26)]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), PROJECT->[c, URL], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"URL":{"terms":{"field":"URL","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q35.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q35.yaml new file mode 100644 index 00000000000..da70cfee61a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q35.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(c=[$2], const=[$0], URL=[$1]) + LogicalAggregate(group=[{0, 1}], c=[COUNT()]) + LogicalProject(const=[$111], URL=[$26]) + LogicalFilter(condition=[IS NOT NULL($26)]) + LogicalProject(EventDate=[$0], URLRegionID=[$1], HasGCLID=[$2], Income=[$3], Interests=[$4], Robotness=[$5], BrowserLanguage=[$6], CounterClass=[$7], BrowserCountry=[$8], OriginalURL=[$9], ClientTimeZone=[$10], RefererHash=[$11], TraficSourceID=[$12], HitColor=[$13], RefererRegionID=[$14], URLCategoryID=[$15], LocalEventTime=[$16], EventTime=[$17], UTMTerm=[$18], AdvEngineID=[$19], UserAgentMinor=[$20], UserAgentMajor=[$21], RemoteIP=[$22], Sex=[$23], JavaEnable=[$24], URLHash=[$25], URL=[$26], ParamOrderID=[$27], OpenstatSourceID=[$28], HTTPError=[$29], SilverlightVersion3=[$30], MobilePhoneModel=[$31], SilverlightVersion4=[$32], SilverlightVersion1=[$33], SilverlightVersion2=[$34], IsDownload=[$35], IsParameter=[$36], CLID=[$37], FlashMajor=[$38], FlashMinor=[$39], UTMMedium=[$40], WatchID=[$41], DontCountHits=[$42], CookieEnable=[$43], HID=[$44], SocialAction=[$45], WindowName=[$46], ConnectTiming=[$47], PageCharset=[$48], IsLink=[$49], IsArtifical=[$50], JavascriptEnable=[$51], ClientEventTime=[$52], DNSTiming=[$53], CodeVersion=[$54], ResponseEndTiming=[$55], FUniqID=[$56], WindowClientHeight=[$57], OpenstatServiceName=[$58], UTMContent=[$59], HistoryLength=[$60], IsOldCounter=[$61], MobilePhone=[$62], SearchPhrase=[$63], FlashMinor2=[$64], SearchEngineID=[$65], IsEvent=[$66], UTMSource=[$67], RegionID=[$68], OpenstatAdID=[$69], UTMCampaign=[$70], GoodEvent=[$71], IsRefresh=[$72], ParamCurrency=[$73], Params=[$74], ResolutionHeight=[$75], ClientIP=[$76], FromTag=[$77], ParamCurrencyID=[$78], ResponseStartTiming=[$79], ResolutionWidth=[$80], SendTiming=[$81], RefererCategoryID=[$82], OpenstatCampaignID=[$83], UserID=[$84], WithHash=[$85], UserAgent=[$86], ParamPrice=[$87], ResolutionDepth=[$88], IsMobile=[$89], Age=[$90], SocialSourceNetworkID=[$91], OpenerName=[$92], OS=[$93], IsNotBounce=[$94], Referer=[$95], NetMinor=[$96], Title=[$97], NetMajor=[$98], IPNetworkID=[$99], FetchTiming=[$100], SocialNetwork=[$101], SocialSourcePage=[$102], CounterID=[$103], WindowClientWidth=[$104], _id=[$105], _index=[$106], _score=[$107], _maxscore=[$108], _sort=[$109], _routing=[$110], const=[1]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], c=[$t1], const=[$t2], URL=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"URL":{"terms":{"field":"URL","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q36.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q36.yaml new file mode 100644 index 00000000000..4f05d895b3a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q36.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(c=[$4], ClientIP=[$0], ClientIP - 1=[$1], ClientIP - 2=[$2], ClientIP - 3=[$3]) + LogicalAggregate(group=[{0, 1, 2, 3}], c=[COUNT()]) + LogicalProject(ClientIP=[$76], ClientIP - 1=[$111], ClientIP - 2=[$112], ClientIP - 3=[$113]) + LogicalFilter(condition=[AND(IS NOT NULL($76), IS NOT NULL($111), IS NOT NULL($112), IS NOT NULL($113))]) + LogicalProject(EventDate=[$0], URLRegionID=[$1], HasGCLID=[$2], Income=[$3], Interests=[$4], Robotness=[$5], BrowserLanguage=[$6], CounterClass=[$7], BrowserCountry=[$8], OriginalURL=[$9], ClientTimeZone=[$10], RefererHash=[$11], TraficSourceID=[$12], HitColor=[$13], RefererRegionID=[$14], URLCategoryID=[$15], LocalEventTime=[$16], EventTime=[$17], UTMTerm=[$18], AdvEngineID=[$19], UserAgentMinor=[$20], UserAgentMajor=[$21], RemoteIP=[$22], Sex=[$23], JavaEnable=[$24], URLHash=[$25], URL=[$26], ParamOrderID=[$27], OpenstatSourceID=[$28], HTTPError=[$29], SilverlightVersion3=[$30], MobilePhoneModel=[$31], SilverlightVersion4=[$32], SilverlightVersion1=[$33], SilverlightVersion2=[$34], IsDownload=[$35], IsParameter=[$36], CLID=[$37], FlashMajor=[$38], FlashMinor=[$39], UTMMedium=[$40], WatchID=[$41], DontCountHits=[$42], CookieEnable=[$43], HID=[$44], SocialAction=[$45], WindowName=[$46], ConnectTiming=[$47], PageCharset=[$48], IsLink=[$49], IsArtifical=[$50], JavascriptEnable=[$51], ClientEventTime=[$52], DNSTiming=[$53], CodeVersion=[$54], ResponseEndTiming=[$55], FUniqID=[$56], WindowClientHeight=[$57], OpenstatServiceName=[$58], UTMContent=[$59], HistoryLength=[$60], IsOldCounter=[$61], MobilePhone=[$62], SearchPhrase=[$63], FlashMinor2=[$64], SearchEngineID=[$65], IsEvent=[$66], UTMSource=[$67], RegionID=[$68], OpenstatAdID=[$69], UTMCampaign=[$70], GoodEvent=[$71], IsRefresh=[$72], ParamCurrency=[$73], Params=[$74], ResolutionHeight=[$75], ClientIP=[$76], FromTag=[$77], ParamCurrencyID=[$78], ResponseStartTiming=[$79], ResolutionWidth=[$80], SendTiming=[$81], RefererCategoryID=[$82], OpenstatCampaignID=[$83], UserID=[$84], WithHash=[$85], UserAgent=[$86], ParamPrice=[$87], ResolutionDepth=[$88], IsMobile=[$89], Age=[$90], SocialSourceNetworkID=[$91], OpenerName=[$92], OS=[$93], IsNotBounce=[$94], Referer=[$95], NetMinor=[$96], Title=[$97], NetMajor=[$98], IPNetworkID=[$99], FetchTiming=[$100], SocialNetwork=[$101], SocialSourcePage=[$102], CounterID=[$103], WindowClientWidth=[$104], _id=[$105], _index=[$106], _score=[$107], _maxscore=[$108], _sort=[$109], _routing=[$110], ClientIP - 1=[-($76, 1)], ClientIP - 2=[-($76, 2)], ClientIP - 3=[-($76, 3)]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[-($t0, $t2)], expr#4=[2], expr#5=[-($t0, $t4)], expr#6=[3], expr#7=[-($t0, $t6)], c=[$t1], ClientIP=[$t0], ClientIP - 1=[$t3], ClientIP - 2=[$t5], ClientIP - 3=[$t7]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->IS NOT NULL($76), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"ClientIP","boost":1.0}},"aggregations":{"ClientIP":{"terms":{"field":"ClientIP","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q37.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q37.yaml new file mode 100644 index 00000000000..71446de8af9 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q37.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(PageViews=[$1], URL=[$0]) + LogicalAggregate(group=[{0}], PageViews=[COUNT()]) + LogicalProject(URL=[$26]) + LogicalFilter(condition=[IS NOT NULL($26)]) + LogicalFilter(condition=[AND(=($103, 62), >=($0, TIMESTAMP('2013-07-01 00:00:00':VARCHAR)), <=($0, TIMESTAMP('2013-07-31 00:00:00':VARCHAR)), =($42, 0), =($72, 0), <>($26, ''))]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[EventDate, URL, DontCountHits, IsRefresh, CounterID], FILTER->AND(=($4, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':VARCHAR..'2013-07-31 00:00:00':VARCHAR]]:VARCHAR), =($2, 0), =($3, 0), <>($1, '')), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},PageViews=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[PageViews, URL], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-31T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"term":{"DontCountHits":{"value":0,"boost":1.0}}},{"term":{"IsRefresh":{"value":0,"boost":1.0}}},{"bool":{"must":[{"exists":{"field":"URL","boost":1.0}}],"must_not":[{"term":{"URL":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["EventDate","URL","DontCountHits","IsRefresh","CounterID"],"excludes":[]},"aggregations":{"URL":{"terms":{"field":"URL","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"PageViews":"desc"},{"_key":"asc"}]},"aggregations":{"PageViews":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q38.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q38.yaml new file mode 100644 index 00000000000..f41ff988614 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q38.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(PageViews=[$1], Title=[$0]) + LogicalAggregate(group=[{0}], PageViews=[COUNT()]) + LogicalProject(Title=[$97]) + LogicalFilter(condition=[IS NOT NULL($97)]) + LogicalFilter(condition=[AND(=($103, 62), >=($0, TIMESTAMP('2013-07-01 00:00:00':VARCHAR)), <=($0, TIMESTAMP('2013-07-31 00:00:00':VARCHAR)), =($42, 0), =($72, 0), <>($97, ''))]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[EventDate, DontCountHits, IsRefresh, Title, CounterID], FILTER->AND(=($4, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':VARCHAR..'2013-07-31 00:00:00':VARCHAR]]:VARCHAR), =($1, 0), =($2, 0), <>($3, '')), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},PageViews=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[PageViews, Title], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-31T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"term":{"DontCountHits":{"value":0,"boost":1.0}}},{"term":{"IsRefresh":{"value":0,"boost":1.0}}},{"bool":{"must":[{"exists":{"field":"Title","boost":1.0}}],"must_not":[{"term":{"Title":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["EventDate","DontCountHits","IsRefresh","Title","CounterID"],"excludes":[]},"aggregations":{"Title":{"terms":{"field":"Title","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"PageViews":"desc"},{"_key":"asc"}]},"aggregations":{"PageViews":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q39.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q39.yaml new file mode 100644 index 00000000000..1366bcd2c31 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q39.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], offset=[1000], fetch=[10]) + LogicalProject(PageViews=[$1], URL=[$0]) + LogicalAggregate(group=[{0}], PageViews=[COUNT()]) + LogicalProject(URL=[$26]) + LogicalFilter(condition=[IS NOT NULL($26)]) + LogicalFilter(condition=[AND(=($103, 62), >=($0, TIMESTAMP('2013-07-01 00:00:00':VARCHAR)), <=($0, TIMESTAMP('2013-07-31 00:00:00':VARCHAR)), =($72, 0), <>($49, 0), =($35, 0))]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableLimit(offset=[1000], fetch=[10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[EventDate, URL, IsDownload, IsLink, IsRefresh, CounterID], FILTER->AND(=($5, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':VARCHAR..'2013-07-31 00:00:00':VARCHAR]]:VARCHAR), =($4, 0), <>($3, 0), =($2, 0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},PageViews=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[PageViews, URL]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-31T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"term":{"IsRefresh":{"value":0,"boost":1.0}}},{"bool":{"must":[{"exists":{"field":"IsLink","boost":1.0}}],"must_not":[{"term":{"IsLink":{"value":0,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"term":{"IsDownload":{"value":0,"boost":1.0}}}],"filter":[{"exists":{"field":"URL","boost":1.0}},{"exists":{"field":"URL","boost":1.0}},{"exists":{"field":"URL","boost":1.0}},{"exists":{"field":"URL","boost":1.0}},{"exists":{"field":"URL","boost":1.0}},{"exists":{"field":"URL","boost":1.0}},{"exists":{"field":"URL","boost":1.0}},{"exists":{"field":"URL","boost":1.0}},{"exists":{"field":"URL","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["EventDate","URL","IsDownload","IsLink","IsRefresh","CounterID"],"excludes":[]},"aggregations":{"URL":{"terms":{"field":"URL","size":1010,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"PageViews":"desc"},{"_key":"asc"}]},"aggregations":{"PageViews":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q4.yaml new file mode 100644 index 00000000000..36063b37709 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q4.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], avg(UserID)=[AVG($0)]) + LogicalProject(UserID=[$84]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},avg(UserID)=AVG($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"avg(UserID)":{"avg":{"field":"UserID"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q40.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q40.yaml new file mode 100644 index 00000000000..6cf4e29f682 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q40.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], offset=[1000], fetch=[10]) + LogicalProject(PageViews=[$5], TraficSourceID=[$0], SearchEngineID=[$1], AdvEngineID=[$2], Src=[$3], Dst=[$4]) + LogicalAggregate(group=[{0, 1, 2, 3, 4}], PageViews=[COUNT()]) + LogicalProject(TraficSourceID=[$12], SearchEngineID=[$65], AdvEngineID=[$19], Src=[CASE(AND(=($65, 0), =($19, 0)), $95, '':VARCHAR)], Dst=[$26]) + LogicalFilter(condition=[AND(=($103, 62), >=($0, TIMESTAMP('2013-07-01 00:00:00':VARCHAR)), <=($0, TIMESTAMP('2013-07-31 00:00:00':VARCHAR)), =($72, 0))]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableLimit(offset=[1000], fetch=[10]) + EnumerableSort(sort0=[$0], dir0=[DESC-nulls-last]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[EventDate, TraficSourceID, AdvEngineID, URL, SearchEngineID, IsRefresh, Referer, CounterID], FILTER->AND(=($7, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':VARCHAR..'2013-07-31 00:00:00':VARCHAR]]:VARCHAR), =($5, 0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2, 3, 4},PageViews=COUNT()), PROJECT->[PageViews, TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-31T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"term":{"IsRefresh":{"value":0,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["EventDate","TraficSourceID","AdvEngineID","URL","SearchEngineID","IsRefresh","Referer","CounterID"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"TraficSourceID":{"terms":{"field":"TraficSourceID","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"SearchEngineID":{"terms":{"field":"SearchEngineID","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"AdvEngineID":{"terms":{"field":"AdvEngineID","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"Src":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQBT3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJTTUFMTElOVCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIlNlYXJjaEVuZ2luZUlEIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiU01BTExJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJBZHZFbmdpbmVJRCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIlJlZmVyZXIiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQE8XsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiQU5EIiwKICAgICAgICAia2luZCI6ICJBTkQiLAogICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgIm9wIjogewogICAgICAgICAgICAibmFtZSI6ICI9IiwKICAgICAgICAgICAgImtpbmQiOiAiRVFVQUxTIiwKICAgICAgICAgICAgInN5bnRheCI6ICJCSU5BUlkiCiAgICAgICAgICB9LAogICAgICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICAgICAgfSwKICAgICAgICAgICAgewogICAgICAgICAgICAgICJsaXRlcmFsIjogMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiPSIsCiAgICAgICAgICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAgICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICAgICAgfSwKICAgICAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICAgICAgewogICAgICAgICAgICAgICJpbnB1dCI6IDEsCiAgICAgICAgICAgICAgIm5hbWUiOiAiJDEiCiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IDAsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9CiAgICAgICAgICBdCiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAiaW5wdXQiOiAyLAogICAgICAibmFtZSI6ICIkMiIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogIiIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAADdAAOU2VhcmNoRW5naW5lSUR+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAVTSE9SVHQAB1JlZmVyZXJ+cQB+AAp0AAZTVFJJTkd0AAtBZHZFbmdpbmVJRHEAfgAMeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":true,"missing_order":"first","order":"asc"}}},{"Dst":{"terms":{"field":"URL","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q41.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q41.yaml new file mode 100644 index 00000000000..60737163de5 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q41.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], offset=[100], fetch=[10]) + LogicalProject(PageViews=[$2], URLHash=[$0], EventDate=[$1]) + LogicalAggregate(group=[{0, 1}], PageViews=[COUNT()]) + LogicalProject(URLHash=[$25], EventDate=[$0]) + LogicalFilter(condition=[AND(IS NOT NULL($25), IS NOT NULL($0))]) + LogicalFilter(condition=[AND(=($103, 62), >=($0, TIMESTAMP('2013-07-01 00:00:00':VARCHAR)), <=($0, TIMESTAMP('2013-07-31 00:00:00':VARCHAR)), =($72, 0), SEARCH($12, Sarg[-1, 6]), =($11, 3594120000172545465))]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableLimit(offset=[100], fetch=[10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[EventDate, RefererHash, TraficSourceID, URLHash, IsRefresh, CounterID], FILTER->AND(=($5, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':VARCHAR..'2013-07-31 00:00:00':VARCHAR]; NULL AS FALSE]:VARCHAR), =($4, 0), SEARCH($2, Sarg[-1, 6]), =($1, 3594120000172545465), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},PageViews=COUNT()), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[PageViews, URLHash, EventDate]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"bool":{"must":[{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-31T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"exists":{"field":"EventDate","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"term":{"IsRefresh":{"value":0,"boost":1.0}}},{"terms":{"TraficSourceID":[-1.0,6.0],"boost":1.0}},{"term":{"RefererHash":{"value":3594120000172545465,"boost":1.0}}},{"exists":{"field":"URLHash","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["EventDate","RefererHash","TraficSourceID","URLHash","IsRefresh","CounterID"],"excludes":[]},"aggregations":{"URLHash|EventDate":{"multi_terms":{"terms":[{"field":"URLHash"},{"field":"EventDate","value_type":"long"}],"size":110,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]},"aggregations":{"PageViews":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q42.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q42.yaml new file mode 100644 index 00000000000..08a95aa3cba --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q42.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], offset=[10000], fetch=[10]) + LogicalProject(PageViews=[$2], WindowClientWidth=[$0], WindowClientHeight=[$1]) + LogicalAggregate(group=[{0, 1}], PageViews=[COUNT()]) + LogicalProject(WindowClientWidth=[$104], WindowClientHeight=[$57]) + LogicalFilter(condition=[AND(IS NOT NULL($104), IS NOT NULL($57))]) + LogicalFilter(condition=[AND(=($103, 62), >=($0, TIMESTAMP('2013-07-01 00:00:00':VARCHAR)), <=($0, TIMESTAMP('2013-07-31 00:00:00':VARCHAR)), =($72, 0), =($42, 0), =($25, 2868770270353813622))]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableLimit(offset=[10000], fetch=[10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[EventDate, URLHash, DontCountHits, WindowClientHeight, IsRefresh, CounterID, WindowClientWidth], FILTER->AND(=($5, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':VARCHAR..'2013-07-31 00:00:00':VARCHAR]]:VARCHAR), =($4, 0), =($2, 0), =($1, 2868770270353813622), IS NOT NULL($6), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},PageViews=COUNT()), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[PageViews, WindowClientWidth, WindowClientHeight]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-31T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"term":{"IsRefresh":{"value":0,"boost":1.0}}},{"term":{"DontCountHits":{"value":0,"boost":1.0}}},{"term":{"URLHash":{"value":2868770270353813622,"boost":1.0}}},{"exists":{"field":"WindowClientWidth","boost":1.0}},{"exists":{"field":"WindowClientHeight","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["EventDate","URLHash","DontCountHits","WindowClientHeight","IsRefresh","CounterID","WindowClientWidth"],"excludes":[]},"aggregations":{"WindowClientWidth|WindowClientHeight":{"multi_terms":{"terms":[{"field":"WindowClientWidth"},{"field":"WindowClientHeight"}],"size":10000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]},"aggregations":{"PageViews":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q43.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q43.yaml new file mode 100644 index 00000000000..98db87536bb --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q43.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], offset=[1000], fetch=[10]) + LogicalProject(PageViews=[$1], M=[$0]) + LogicalAggregate(group=[{0}], PageViews=[COUNT()]) + LogicalProject(M=[DATE_FORMAT($17, '%Y-%m-%d %H:00:00':VARCHAR)]) + LogicalFilter(condition=[AND(=($103, 62), >=($0, TIMESTAMP('2013-07-01 00:00:00':VARCHAR)), <=($0, TIMESTAMP('2013-07-15 00:00:00':VARCHAR)), =($72, 0), =($42, 0))]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + EnumerableCalc(expr#0..1=[{inputs}], PageViews=[$t1], M=[$t0]) + EnumerableLimit(fetch=[10000]) + EnumerableLimit(offset=[1000], fetch=[10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[EventDate, EventTime, DontCountHits, IsRefresh, CounterID], FILTER->AND(=($4, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':VARCHAR..'2013-07-15 00:00:00':VARCHAR]]:VARCHAR), =($3, 0), =($2, 0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},PageViews=COUNT()), SORT->[0 ASC FIRST], LIMIT->[10 from 1000]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-15T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"term":{"IsRefresh":{"value":0,"boost":1.0}}},{"term":{"DontCountHits":{"value":0,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["EventDate","EventTime","DontCountHits","IsRefresh","CounterID"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1010,"sources":[{"M":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAt3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiRXZlbnRUaW1lIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AhN7CiAgIm9wIjogewogICAgIm5hbWUiOiAiREFURV9GT1JNQVQiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICIlWS0lbS0lZCAlSDowMDowMCIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAicHJlY2lzaW9uIjogLTEKICB9LAogICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAiZHluYW1pYyI6IGZhbHNlCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AAlFdmVudFRpbWVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRlVHlwZZ4tUq4QfcqvAgABTAAHZm9ybWF0c3QAEExqYXZhL3V0aWwvTGlzdDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3QAD0xqYXZhL3V0aWwvTWFwO3hwfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAJVElNRVNUQU1QfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4AEnQABERhdGVzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABl4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+ABsAAAAAc3IAE2phdmEudXRpbC5BcnJheUxpc3R4gdIdmcdhnQMAAUkABHNpemV4cAAAAAN3BAAAAAN0ABN5eXl5LU1NLWRkIEhIOm1tOnNzdAAZc3RyaWN0X2RhdGVfb3B0aW9uYWxfdGltZXQADGVwb2NoX21pbGxpc3h4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q5.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q5.yaml new file mode 100644 index 00000000000..3edc0813519 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q5.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], dc(UserID)=[COUNT(DISTINCT $0)]) + LogicalProject(UserID=[$84]) + LogicalFilter(condition=[IS NOT NULL($84)]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->IS NOT NULL($84), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},dc(UserID)=COUNT(DISTINCT $0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"UserID","boost":1.0}},"aggregations":{"dc(UserID)":{"cardinality":{"field":"UserID"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q6.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q6.yaml new file mode 100644 index 00000000000..9a8d579de98 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q6.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], dc(SearchPhrase)=[COUNT(DISTINCT $0)]) + LogicalProject(SearchPhrase=[$63]) + LogicalFilter(condition=[IS NOT NULL($63)]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->IS NOT NULL($63), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},dc(SearchPhrase)=COUNT(DISTINCT $0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"SearchPhrase","boost":1.0}},"aggregations":{"dc(SearchPhrase)":{"cardinality":{"field":"SearchPhrase"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q7.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q7.yaml new file mode 100644 index 00000000000..5d87e2daaab --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q7.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], min(EventDate)=[MIN($0)], max(EventDate)=[MAX($0)]) + LogicalProject(EventDate=[$0]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},min(EventDate)=MIN($0),max(EventDate)=MAX($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"min(EventDate)":{"min":{"field":"EventDate"}},"max(EventDate)":{"max":{"field":"EventDate"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q8.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q8.yaml new file mode 100644 index 00000000000..f57500d3809 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q8.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last]) + LogicalProject(count()=[$1], AdvEngineID=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(AdvEngineID=[$19]) + LogicalFilter(condition=[IS NOT NULL($19)]) + LogicalFilter(condition=[<>($19, 0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER-><>($19, 0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[count(), AdvEngineID], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"AdvEngineID","boost":1.0}}],"must_not":[{"term":{"AdvEngineID":{"value":0,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"AdvEngineID":{"terms":{"field":"AdvEngineID","size":10000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"count()":"desc"},{"_key":"asc"}]},"aggregations":{"count()":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q9.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q9.yaml new file mode 100644 index 00000000000..5e6bc1617c5 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q9.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(u=[$1], RegionID=[$0]) + LogicalAggregate(group=[{0}], u=[COUNT(DISTINCT $1)]) + LogicalProject(RegionID=[$68], UserID=[$84]) + LogicalFilter(condition=[IS NOT NULL($68)]) + CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},u=COUNT(DISTINCT $1)), PROJECT->[u, RegionID], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"RegionID":{"terms":{"field":"RegionID","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"u":"desc"},{"_key":"asc"}]},"aggregations":{"u":{"cardinality":{"field":"UserID"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/composite_date_histogram_daily.yaml b/integ-test/src/test/resources/expectedOutput/calcite/composite_date_histogram_daily.yaml new file mode 100644 index 00000000000..9b69c67b74c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/composite_date_histogram_daily.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], span(`@timestamp`,1d)=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(span(`@timestamp`,1d)=[SPAN($17, 1, 'd')]) + LogicalFilter(condition=[IS NOT NULL($17)]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2022-12-30 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-07 12:00:00':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[@timestamp], FILTER->SEARCH($0, Sarg[['2022-12-30 00:00:00':VARCHAR..'2023-01-07 12:00:00':VARCHAR); NULL AS FALSE]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(`@timestamp`,1d)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2022-12-30T00:00:00.000Z","to":"2023-01-07T12:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"exists":{"field":"@timestamp","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["@timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"span(`@timestamp`,1d)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/composite_terms.yaml b/integ-test/src/test/resources/expectedOutput/calcite/composite_terms.yaml new file mode 100644 index 00000000000..8720f023f80 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/composite_terms.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$1], sort1=[$2], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$1], sort1=[$2], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first]) + LogicalProject(count()=[$2], process.name=[$0], cloud.region=[$1]) + LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) + LogicalProject(process.name=[$7], cloud.region=[$14]) + LogicalFilter(condition=[AND(IS NOT NULL($7), IS NOT NULL($14))]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-02 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-02 10:00:00':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[process.name, cloud.region, @timestamp], FILTER->SEARCH($2, Sarg[['2023-01-02 00:00:00':VARCHAR..'2023-01-02 10:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), process.name, cloud.region], SORT->[1 DESC LAST, 2 ASC FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-02T00:00:00.000Z","to":"2023-01-02T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["process.name","cloud.region","@timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":false,"order":"desc"}}},{"cloud.region":{"terms":{"field":"cloud.region","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/composite_terms_keyword.yaml b/integ-test/src/test/resources/expectedOutput/calcite/composite_terms_keyword.yaml new file mode 100644 index 00000000000..ac251d900f0 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/composite_terms_keyword.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$1], sort1=[$2], sort2=[$3], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first], dir2=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$1], sort1=[$2], sort2=[$3], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first], dir2=[ASC-nulls-first]) + LogicalProject(count()=[$3], process.name=[$0], cloud.region=[$1], aws.cloudwatch.log_stream=[$2]) + LogicalAggregate(group=[{0, 1, 2}], count()=[COUNT()]) + LogicalProject(process.name=[$7], cloud.region=[$14], aws.cloudwatch.log_stream=[$34]) + LogicalFilter(condition=[AND(IS NOT NULL($7), IS NOT NULL($14), IS NOT NULL($34))]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-02 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-02 10:00:00':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[process.name, cloud.region, @timestamp, aws.cloudwatch.log_stream], FILTER->SEARCH($2, Sarg[['2023-01-02 00:00:00':VARCHAR..'2023-01-02 10:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2},count()=COUNT()), PROJECT->[count(), process.name, cloud.region, aws.cloudwatch.log_stream], SORT->[1 DESC LAST, 2 ASC FIRST, 3 ASC FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-02T00:00:00.000Z","to":"2023-01-02T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["process.name","cloud.region","@timestamp","aws.cloudwatch.log_stream"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":false,"order":"desc"}}},{"cloud.region":{"terms":{"field":"cloud.region","missing_bucket":false,"order":"asc"}}},{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/date_histogram_hourly_agg.yaml b/integ-test/src/test/resources/expectedOutput/calcite/date_histogram_hourly_agg.yaml new file mode 100644 index 00000000000..06361ea27e8 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/date_histogram_hourly_agg.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], span(`@timestamp`,1h)=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(span(`@timestamp`,1h)=[SPAN($17, 1, 'h')]) + LogicalFilter(condition=[IS NOT NULL($17)]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[@timestamp], FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(`@timestamp`,1h)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"@timestamp","boost":1.0}},"_source":{"includes":["@timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"span(`@timestamp`,1h)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1h"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/date_histogram_minute_agg.yaml b/integ-test/src/test/resources/expectedOutput/calcite/date_histogram_minute_agg.yaml new file mode 100644 index 00000000000..c715c2c2a42 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/date_histogram_minute_agg.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], span(`@timestamp`,1m)=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(span(`@timestamp`,1m)=[SPAN($17, 1, 'm')]) + LogicalFilter(condition=[IS NOT NULL($17)]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[@timestamp], FILTER->SEARCH($0, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR); NULL AS FALSE]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(`@timestamp`,1m)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"exists":{"field":"@timestamp","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["@timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"span(`@timestamp`,1m)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/default.yaml b/integ-test/src/test/resources/expectedOutput/calcite/default.yaml new file mode 100644 index 00000000000..59e68e48769 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/default.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_timestamp.yaml new file mode 100644 index 00000000000..7e14abeeef2 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_timestamp.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$7], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$17], dir0=[DESC-nulls-last], fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], SORT->[{ + "@timestamp" : { + "order" : "desc", + "missing" : "_last" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_timestamp_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_timestamp_can_match_shortcut.yaml new file mode 100644 index 00000000000..13239b869cc --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_timestamp_can_match_shortcut.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$7], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$17], dir0=[DESC-nulls-last], fetch=[10]) + LogicalFilter(condition=[query_string(MAP('query', 'process.name:kernel':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->query_string(MAP('query', 'process.name:kernel':VARCHAR)), SORT->[{ + "@timestamp" : { + "order" : "desc", + "missing" : "_last" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_timestamp_no_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_timestamp_no_can_match_shortcut.yaml new file mode 100644 index 00000000000..13239b869cc --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_timestamp_no_can_match_shortcut.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$7], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$17], dir0=[DESC-nulls-last], fetch=[10]) + LogicalFilter(condition=[query_string(MAP('query', 'process.name:kernel':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->query_string(MAP('query', 'process.name:kernel':VARCHAR)), SORT->[{ + "@timestamp" : { + "order" : "desc", + "missing" : "_last" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_with_after_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_with_after_timestamp.yaml new file mode 100644 index 00000000000..7e14abeeef2 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_with_after_timestamp.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$7], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$17], dir0=[DESC-nulls-last], fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], SORT->[{ + "@timestamp" : { + "order" : "desc", + "missing" : "_last" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by4.yaml index e56eb5ad662..77fc6c6eadf 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by4.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by4.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(gender=[$4], account_number=[$0]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT(),count(account_number)=COUNT($1)), PROJECT->[count(), count(account_number), gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count()":{"value_count":{"field":"_index"}},"count(account_number)":{"value_count":{"field":"account_number"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT(),count(account_number)=COUNT($1)), PROJECT->[count(), count(account_number), gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(account_number)":{"value_count":{"field":"account_number"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml index a385eaf690d..9d452158542 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(gender=[$4], b_1=[+($3, 1)], $f3=[POWER($3, 2)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(b_1)=COUNT($1),c3=COUNT($2)), PROJECT->[count(b_1), c3, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(b_1)":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBAXsKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDMsCiAgICAgICJuYW1lIjogIiQzIgogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAReHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABx4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB4AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABJ+cQB+AAt0AAZTVFJJTkd+cQB+ABh0AAdLZXl3b3JkcQB+AB14dAAHYWRkcmVzc3NxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgANdAAGZ2VuZGVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAEY2l0eXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGVtcGxveWVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAFc3RhdGVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AANhZ2VxAH4ADXQABWVtYWlsc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIbGFzdG5hbWVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h4AHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}},"c3":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBEXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJQT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiAzLAogICAgICAibmFtZSI6ICIkMyIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgAXamF2YS51dGlsLkxpbmtlZEhhc2hNYXA0wE5cEGzA+wIAAVoAC2FjY2Vzc09yZGVyeHIAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAt0AA5hY2NvdW50X251bWJlcn5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABExPTkd0AAlmaXJzdG5hbWVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AEXhwfnEAfgALdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AAx0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAceHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAeAAAAAHNxAH4AAAAAAAN3BAAAAAJ0AAdrZXl3b3Jkc3EAfgASfnEAfgALdAAGU1RSSU5HfnEAfgAYdAAHS2V5d29yZHEAfgAdeHQAB2FkZHJlc3NzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAAAeHQAB2JhbGFuY2VxAH4ADXQABmdlbmRlcnNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQABGNpdHlzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AAhlbXBsb3llcnNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQABXN0YXRlc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAADYWdlcQB+AA10AAVlbWFpbHNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGxhc3RuYW1lc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4eAB4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(b_1)=COUNT($1),c3=COUNT($2)), PROJECT->[count(b_1), c3, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(b_1)":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAfnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AQF7CiAgIm9wIjogewogICAgIm5hbWUiOiAiKyIsCiAgICAia2luZCI6ICJQTFVTIiwKICAgICJzeW50YXgiOiAiQklOQVJZIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiAwLAogICAgICAibmFtZSI6ICIkMCIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQAB2JhbGFuY2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05HeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}},"c3":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAfnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0ARF7CiAgIm9wIjogewogICAgIm5hbWUiOiAiUE9XRVIiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AAdiYWxhbmNlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3h4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_group_merge.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_group_merge.yaml new file mode 100644 index 00000000000..aaf4f2017cb --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_group_merge.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$4], age1=[$0], age2=[$1], age3=[$2], age=[$3]) + LogicalAggregate(group=[{0, 1, 2, 3}], count()=[COUNT()]) + LogicalProject(age1=[*($8, 10)], age2=[+($8, 10)], age3=[10], age=[$8]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10], expr#3=[*($t0, $t2)], expr#4=[+($t0, $t2)], count()=[$t1], age1=[$t3], age2=[$t4], age3=[$t2], age=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_on_window.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_on_window.json deleted file mode 100644 index 20aaa1a6f9b..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_on_window.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], patterns_field=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(patterns_field=[SAFE_CAST(ITEM(PATTERN_PARSER($2, pattern($2, 10, 100000, false) OVER (), false), 'pattern'))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], patterns_field=[$t0])\n EnumerableAggregate(group=[{0}], count()=[COUNT()])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[false], expr#3=[PATTERN_PARSER($t0, $t1, $t2)], expr#4=['pattern'], expr#5=[ITEM($t3, $t4)], expr#6=[SAFE_CAST($t5)], patterns_field=[$t6])\n EnumerableWindow(window#0=[window(aggs [pattern($0, $1, $2, $3)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[address]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"address\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_on_window.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_on_window.yaml new file mode 100644 index 00000000000..bc2c9debf47 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_on_window.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], patterns_field=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(patterns_field=[SAFE_CAST(ITEM(PATTERN_PARSER($2, pattern($2, 10, 100000, false) OVER (), false), 'pattern'))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], patterns_field=[$t0]) + EnumerableAggregate(group=[{0}], count()=[COUNT()]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[false], expr#3=[PATTERN_PARSER($t0, $t1, $t2)], expr#4=['pattern'], expr#5=[ITEM($t3, $t4)], expr#6=[SAFE_CAST($t5)], patterns_field=[$t6]) + EnumerableWindow(window#0=[window(aggs [pattern($0, $1, $2, $3)])], constants=[[10, 100000, false]]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[address]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["address"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_timestamp_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_timestamp_push.json deleted file mode 100644 index e0c197e8efd..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_timestamp_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], fetch=[3])\n LogicalProject(count()=[$1], t=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(t=[UNIX_TIMESTAMP($3)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT->[0 ASC FIRST], PROJECT->[count(), t], LIMIT->3, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":3,\"sources\":[{\"t\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0AWd7CiAgIm9wIjogewogICAgIm5hbWUiOiAiVU5JWF9USU1FU1RBTVAiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMywKICAgICAgIm5hbWUiOiAiJDMiCiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJET1VCTEUiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfXQACmZpZWxkVHlwZXNzcgAXamF2YS51dGlsLkxpbmtlZEhhc2hNYXA0wE5cEGzA+wIAAVoAC2FjY2Vzc09yZGVyeHIAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAAGHcIAAAAIAAAAA10AA5hY2NvdW50X251bWJlcn5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABExPTkd0AAlmaXJzdG5hbWV+cQB+AAt0AAZTVFJJTkd0AAdhZGRyZXNzc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+ABR4cH5xAH4AC3QAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgAMdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AH3hwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AIQAAAABzcQB+AAAAAAADdwQAAAAAeHQACWJpcnRoZGF0ZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGVUeXBlni1SrhB9yq8CAAFMAAdmb3JtYXRzdAAQTGphdmEvdXRpbC9MaXN0O3hxAH4AFX5xAH4AC3QACVRJTUVTVEFNUH5xAH4AG3QABERhdGVxAH4AIHNxAH4AAAAAAAF3BAAAAAB4dAAGZ2VuZGVyc3EAfgATcQB+ABlxAH4AHHEAfgAgc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABVxAH4AEH5xAH4AG3QAB0tleXdvcmRxAH4AIHh0AARjaXR5cQB+ABB0AAhsYXN0bmFtZXEAfgAQdAAHYmFsYW5jZXEAfgANdAAIZW1wbG95ZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABXN0YXRlc3EAfgATcQB+ABlxAH4AHHEAfgAgc3EAfgAAAAAAA3cEAAAAAnEAfgAxcQB+ADJ4dAADYWdlfnEAfgALdAAHSU5URUdFUnQABWVtYWlsc3EAfgATcQB+ABlxAH4AHHEAfgAgcQB+ACR0AARtYWxlfnEAfgALdAAHQk9PTEVBTngAeA==\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_timestamp_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_timestamp_push.yaml new file mode 100644 index 00000000000..bfaf02ad47c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_timestamp_push.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], fetch=[3]) + LogicalProject(count()=[$1], t=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(t=[UNIX_TIMESTAMP($3)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), t], SORT->[1 ASC FIRST], LIMIT->3, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":3,"sources":[{"t":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAt3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AWd7CiAgIm9wIjogewogICAgIm5hbWUiOiAiVU5JWF9USU1FU1RBTVAiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJET1VCTEUiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQACWJpcnRoZGF0ZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGVUeXBlni1SrhB9yq8CAAFMAAdmb3JtYXRzdAAQTGphdmEvdXRpbC9MaXN0O3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzdAAPTGphdmEvdXRpbC9NYXA7eHB+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAlUSU1FU1RBTVB+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgASdAAERGF0ZXNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGXhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGwAAAABzcQB+AAAAAAABdwQAAAAAeHh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":true,"value_type":"double","missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_udt_arg_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_udt_arg_push.json deleted file mode 100644 index 6d5f44e2da2..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_udt_arg_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(t,1d)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(t,1d)=[SPAN($19, 1, 'd')])\n LogicalFilter(condition=[IS NOT NULL($19)])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], t=[DATE_ADD($3, 1:INTERVAL DAY)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[SCRIPT->IS NOT NULL(DATE_ADD($3, 1:INTERVAL DAY)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(t,1d)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0AyV7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiREFURV9BREQiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMywKICAgICAgICAgICJuYW1lIjogIiQzIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVSVkFMX0RBWSIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogMTAsCiAgICAgICAgICAgICJzY2FsZSI6IDYKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInVkdCI6ICJFWFBSX1RJTUVTVEFNUCIsCiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfSwKICAgICAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAABh3CAAAACAAAAANdAAOYWNjb3VudF9udW1iZXJ+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05HdAAJZmlyc3RuYW1lfnEAfgAKdAAGU1RSSU5HdAAHYWRkcmVzc3NyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgATeHB+cQB+AAp0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4AC3QABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+AB54cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+ACAAAAAAc3EAfgAAAAAAA3cEAAAAAHh0AAliaXJ0aGRhdGVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRlVHlwZZ4tUq4QfcqvAgABTAAHZm9ybWF0c3QAEExqYXZhL3V0aWwvTGlzdDt4cQB+ABR+cQB+AAp0AAlUSU1FU1RBTVB+cQB+ABp0AAREYXRlcQB+AB9zcQB+AAAAAAABdwQAAAAAeHQABmdlbmRlcnNxAH4AEnEAfgAYcQB+ABtxAH4AH3NxAH4AAAAAAAN3BAAAAAJ0AAdrZXl3b3Jkc3EAfgAUcQB+AA9+cQB+ABp0AAdLZXl3b3JkcQB+AB94dAAEY2l0eXEAfgAPdAAIbGFzdG5hbWVxAH4AD3QAB2JhbGFuY2VxAH4ADHQACGVtcGxveWVyc3EAfgAScQB+ABhxAH4AG3EAfgAfcQB+ACN0AAVzdGF0ZXNxAH4AEnEAfgAYcQB+ABtxAH4AH3NxAH4AAAAAAAN3BAAAAAJxAH4AMHEAfgAxeHQAA2FnZX5xAH4ACnQAB0lOVEVHRVJ0AAVlbWFpbHNxAH4AEnEAfgAYcQB+ABtxAH4AH3EAfgAjdAAEbWFsZX5xAH4ACnQAB0JPT0xFQU54eA==\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(t,1d)\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0BQN7CiAgIm9wIjogewogICAgIm5hbWUiOiAiU1BBTiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiREFURV9BREQiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMywKICAgICAgICAgICJuYW1lIjogIiQzIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVSVkFMX0RBWSIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogMTAsCiAgICAgICAgICAgICJzY2FsZSI6IDYKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInVkdCI6ICJFWFBSX1RJTUVTVEFNUCIsCiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfSwKICAgICAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogImQiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogMQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICJwcmVjaXNpb24iOiAtMQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfXQACmZpZWxkVHlwZXNzcgAXamF2YS51dGlsLkxpbmtlZEhhc2hNYXA0wE5cEGzA+wIAAVoAC2FjY2Vzc09yZGVyeHIAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAAGHcIAAAAIAAAAA10AA5hY2NvdW50X251bWJlcn5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABExPTkd0AAlmaXJzdG5hbWV+cQB+AAt0AAZTVFJJTkd0AAdhZGRyZXNzc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+ABR4cH5xAH4AC3QAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgAMdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AH3hwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AIQAAAABzcQB+AAAAAAADdwQAAAAAeHQACWJpcnRoZGF0ZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGVUeXBlni1SrhB9yq8CAAFMAAdmb3JtYXRzdAAQTGphdmEvdXRpbC9MaXN0O3hxAH4AFX5xAH4AC3QACVRJTUVTVEFNUH5xAH4AG3QABERhdGVxAH4AIHNxAH4AAAAAAAF3BAAAAAB4dAAGZ2VuZGVyc3EAfgATcQB+ABlxAH4AHHEAfgAgc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABVxAH4AEH5xAH4AG3QAB0tleXdvcmRxAH4AIHh0AARjaXR5cQB+ABB0AAhsYXN0bmFtZXEAfgAQdAAHYmFsYW5jZXEAfgANdAAIZW1wbG95ZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABXN0YXRlc3EAfgATcQB+ABlxAH4AHHEAfgAgc3EAfgAAAAAAA3cEAAAAAnEAfgAxcQB+ADJ4dAADYWdlfnEAfgALdAAHSU5URUdFUnQABWVtYWlsc3EAfgATcQB+ABlxAH4AHHEAfgAgcQB+ACR0AARtYWxlfnEAfgALdAAHQk9PTEVBTngAeA==\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"missing_bucket\":true,\"value_type\":\"long\",\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_udt_arg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_udt_arg_push.yaml new file mode 100644 index 00000000000..e0d54375e15 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_udt_arg_push.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], span(t,1d)=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(span(t,1d)=[SPAN($19, 1, 'd')]) + LogicalFilter(condition=[IS NOT NULL($19)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], t=[DATE_ADD($3, 1:INTERVAL DAY)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[SCRIPT->IS NOT NULL(DATE_ADD($3, 1:INTERVAL DAY)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(t,1d)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAt3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AyV7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiREFURV9BREQiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVSVkFMX0RBWSIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogMTAsCiAgICAgICAgICAgICJzY2FsZSI6IDYKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInVkdCI6ICJFWFBSX1RJTUVTVEFNUCIsCiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfSwKICAgICAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAAJYmlydGhkYXRlc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0ZVR5cGWeLVKuEH3KrwIAAUwAB2Zvcm1hdHN0ABBMamF2YS91dGlsL0xpc3Q7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXN0AA9MamF2YS91dGlsL01hcDt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQACVRJTUVTVEFNUH5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+ABJ0AAREYXRlc3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAZeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAbAAAAAHNxAH4AAAAAAAF3BAAAAAB4eHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(t,1d)":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAt3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BQN7CiAgIm9wIjogewogICAgIm5hbWUiOiAiU1BBTiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiREFURV9BREQiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVSVkFMX0RBWSIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogMTAsCiAgICAgICAgICAgICJzY2FsZSI6IDYKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInVkdCI6ICJFWFBSX1RJTUVTVEFNUCIsCiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfSwKICAgICAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogImQiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogMQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICJwcmVjaXNpb24iOiAtMQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQACWJpcnRoZGF0ZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGVUeXBlni1SrhB9yq8CAAFMAAdmb3JtYXRzdAAQTGphdmEvdXRpbC9MaXN0O3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzdAAPTGphdmEvdXRpbC9NYXA7eHB+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAlUSU1FU1RBTVB+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgASdAAERGF0ZXNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGXhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGwAAAABzcQB+AAAAAAABdwQAAAAAeHh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":true,"value_type":"long","missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure1.yaml new file mode 100644 index 00000000000..b837e4968d4 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure1.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC-nulls-first]) + LogicalProject(count()=[$1], state=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(state=[$7]) + LogicalFilter(condition=[IS NOT NULL($7)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT_AGG_METRICS->[1 ASC FIRST], PROJECT->[count(), state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"state":{"terms":{"field":"state.keyword","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"count()":"asc"},{"_key":"asc"}]},"aggregations":{"count()":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure2.yaml new file mode 100644 index 00000000000..1808eba1f08 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure2.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last]) + LogicalProject(sum=[$1], state=[$0]) + LogicalAggregate(group=[{0}], sum=[SUM($1)]) + LogicalProject(state=[$7], balance=[$3]) + LogicalFilter(condition=[IS NOT NULL($7)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum=SUM($0)), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[sum, state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"state":{"terms":{"field":"state.keyword","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"sum":"desc"},{"_key":"asc"}]},"aggregations":{"sum":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure3.yaml new file mode 100644 index 00000000000..a40c5cec466 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure3.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last]) + LogicalProject(cnt=[$1], span(birthdate,1d)=[$0]) + LogicalAggregate(group=[{0}], cnt=[COUNT()]) + LogicalProject(span(birthdate,1d)=[SPAN($3, 1, 'd')]) + LogicalFilter(condition=[IS NOT NULL($3)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[birthdate], FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},cnt=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[cnt, span(birthdate,1d)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"birthdate","boost":1.0}},"_source":{"includes":["birthdate"],"excludes":[]},"aggregations":{"span(birthdate,1d)":{"date_histogram":{"field":"birthdate","fixed_interval":"1d","offset":0,"order":[{"cnt":"desc"},{"_key":"asc"}],"keyed":false,"min_doc_count":0},"aggregations":{"cnt":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml new file mode 100644 index 00000000000..74ff751bcef --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last]) + LogicalProject(sum(balance)=[$1], span(age,5)=[$0]) + LogicalAggregate(group=[{1}], sum(balance)=[SUM($0)]) + LogicalProject(balance=[$7], span(age,5)=[SPAN($10, 5, null:NULL)]) + LogicalFilter(condition=[IS NOT NULL($10)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[balance, age], FILTER->IS NOT NULL($1), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum(balance)=SUM($0)), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[sum(balance), span(age,5)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"age","boost":1.0}},"_source":{"includes":["balance","age"],"excludes":[]},"aggregations":{"span(age,5)":{"histogram":{"field":"age","interval":5.0,"offset":0.0,"order":[{"sum(balance)":"desc"},{"_key":"asc"}],"keyed":false,"min_doc_count":0},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_terms.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_terms.yaml new file mode 100644 index 00000000000..87afd801267 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_terms.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC-nulls-first]) + LogicalProject(count()=[$2], gender=[$0], state=[$1]) + LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) + LogicalProject(gender=[$4], state=[$7]) + LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), SORT_AGG_METRICS->[2 ASC FIRST], PROJECT->[count(), gender, state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"gender|state":{"multi_terms":{"terms":[{"field":"gender.keyword"},{"field":"state.keyword"}],"size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]},"aggregations":{"count()":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_metrics1.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_metrics1.json deleted file mode 100644 index 02352aa32e2..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_metrics1.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$0], dir0=[ASC-nulls-first])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n LogicalFilter(condition=[IS NOT NULL($7)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($7), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"state\",\"boost\":1.0}},\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":false,\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_metrics2.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_metrics2.json deleted file mode 100644 index a40a5e51c16..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_metrics2.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$0], dir0=[ASC-nulls-first])\n LogicalProject(count()=[$2], gender=[$0], state=[$1])\n LogicalAggregate(group=[{0, 1}], count()=[COUNT()])\n LogicalProject(gender=[$4], state=[$7])\n LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->AND(IS NOT NULL($4), IS NOT NULL($7)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), gender, state]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"must\":[{\"exists\":{\"field\":\"gender\",\"boost\":1.0}},{\"exists\":{\"field\":\"state\",\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":false,\"order\":\"asc\"}}},{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":false,\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_distinct_count_approx_enhancement.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_distinct_count_approx_enhancement.json new file mode 100644 index 00000000000..a8d07145562 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_distinct_count_approx_enhancement.json @@ -0,0 +1,6 @@ +{ + "calcite":{ + "logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(distinct_states=[$1], gender=[$0])\n LogicalAggregate(group=[{0}], distinct_states=[DISTINCT_COUNT_APPROX($1)])\n LogicalProject(gender=[$4], state=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical":"CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},distinct_states=DISTINCT_COUNT_APPROX($1)), PROJECT->[distinct_states, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"distinct_states\":{\"cardinality\":{\"field\":\"state.keyword\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + } +} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.json deleted file mode 100644 index 989f996b26e..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(sum=[$2], len=[$0], gender=[$1])\n LogicalAggregate(group=[{0, 1}], sum=[SUM($2)])\n LogicalProject(len=[CHAR_LENGTH($4)], gender=[$4], $f3=[+($7, 100)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},sum=SUM($2)), PROJECT->[sum, len, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"len\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0AKZ7CiAgIm9wIjogewogICAgIm5hbWUiOiAiQ0hBUl9MRU5HVEgiLAogICAgImtpbmQiOiAiQ0hBUl9MRU5HVEgiLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogNCwKICAgICAgIm5hbWUiOiAiJDQiCiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAYdwgAAAAgAAAADXQADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZX5xAH4AC3QABlNUUklOR3QAB2FkZHJlc3NzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AFHhwfnEAfgALdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AAx0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAfeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAhAAAAAHNxAH4AAAAAAAN3BAAAAAB4dAAJYmlydGhkYXRlc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0ZVR5cGWeLVKuEH3KrwIAAUwAB2Zvcm1hdHN0ABBMamF2YS91dGlsL0xpc3Q7eHEAfgAVfnEAfgALdAAJVElNRVNUQU1QfnEAfgAbdAAERGF0ZXEAfgAgc3EAfgAAAAAAAXcEAAAAAHh0AAZnZW5kZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AFXEAfgAQfnEAfgAbdAAHS2V5d29yZHEAfgAgeHQABGNpdHlxAH4AEHQACGxhc3RuYW1lcQB+ABB0AAdiYWxhbmNlcQB+AA10AAhlbXBsb3llcnNxAH4AE3EAfgAZcQB+ABxxAH4AIHEAfgAkdAAFc3RhdGVzcQB+ABNxAH4AGXEAfgAccQB+ACBzcQB+AAAAAAADdwQAAAACcQB+ADFxAH4AMnh0AANhZ2V+cQB+AAt0AAdJTlRFR0VSdAAFZW1haWxzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABG1hbGV+cQB+AAt0AAdCT09MRUFOeAB4\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"sum\":{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0AQN7CiAgIm9wIjogewogICAgIm5hbWUiOiAiKyIsCiAgICAia2luZCI6ICJQTFVTIiwKICAgICJzeW50YXgiOiAiQklOQVJZIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiA3LAogICAgICAibmFtZSI6ICIkNyIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogMTAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAYdwgAAAAgAAAADXQADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZX5xAH4AC3QABlNUUklOR3QAB2FkZHJlc3NzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AFHhwfnEAfgALdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AAx0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAfeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAhAAAAAHNxAH4AAAAAAAN3BAAAAAB4dAAJYmlydGhkYXRlc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0ZVR5cGWeLVKuEH3KrwIAAUwAB2Zvcm1hdHN0ABBMamF2YS91dGlsL0xpc3Q7eHEAfgAVfnEAfgALdAAJVElNRVNUQU1QfnEAfgAbdAAERGF0ZXEAfgAgc3EAfgAAAAAAAXcEAAAAAHh0AAZnZW5kZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AFXEAfgAQfnEAfgAbdAAHS2V5d29yZHEAfgAgeHQABGNpdHlxAH4AEHQACGxhc3RuYW1lcQB+ABB0AAdiYWxhbmNlcQB+AA10AAhlbXBsb3llcnNxAH4AE3EAfgAZcQB+ABxxAH4AIHEAfgAkdAAFc3RhdGVzcQB+ABNxAH4AGXEAfgAccQB+ACBzcQB+AAAAAAADdwQAAAACcQB+ADFxAH4AMnh0AANhZ2V+cQB+AAt0AAdJTlRFR0VSdAAFZW1haWxzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABG1hbGV+cQB+AAt0AAdCT09MRUFOeAB4\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml new file mode 100644 index 00000000000..fcbc002565c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(sum=[$2], len=[$0], gender=[$1]) + LogicalAggregate(group=[{0, 1}], sum=[SUM($2)]) + LogicalProject(len=[CHAR_LENGTH($4)], gender=[$4], $f3=[+($7, 100)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CHAR_LENGTH($t0)], sum=[$t1], len=[$t2], gender=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum=SUM($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAfnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AQN7CiAgIm9wIjogewogICAgIm5hbWUiOiAiKyIsCiAgICAia2luZCI6ICJQTFVTIiwKICAgICJzeW50YXgiOiAiQklOQVJZIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiAwLAogICAgICAibmFtZSI6ICIkMCIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogMTAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAAHYmFsYW5jZX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABExPTkd4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml index dd8a0fac298..e87eb6d895c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0Ac97CiAgIm9wIjogewogICAgIm5hbWUiOiAiRElWSURFIiwKICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDcsCiAgICAgICJuYW1lIjogIiQ3IgogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAxMDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdLAogICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAibnVsbGFibGUiOiB0cnVlCiAgfSwKICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgImR5bmFtaWMiOiBmYWxzZQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAYdwgAAAAgAAAADXQADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZX5xAH4AC3QABlNUUklOR3QAB2FkZHJlc3NzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AFHhwfnEAfgALdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AAx0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAfeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAhAAAAAHNxAH4AAAAAAAN3BAAAAAB4dAAJYmlydGhkYXRlc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0ZVR5cGWeLVKuEH3KrwIAAUwAB2Zvcm1hdHN0ABBMamF2YS91dGlsL0xpc3Q7eHEAfgAVfnEAfgALdAAJVElNRVNUQU1QfnEAfgAbdAAERGF0ZXEAfgAgc3EAfgAAAAAAAXcEAAAAAHh0AAZnZW5kZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AFXEAfgAQfnEAfgAbdAAHS2V5d29yZHEAfgAgeHQABGNpdHlxAH4AEHQACGxhc3RuYW1lcQB+ABB0AAdiYWxhbmNlcQB+AA10AAhlbXBsb3llcnNxAH4AE3EAfgAZcQB+ABxxAH4AIHEAfgAkdAAFc3RhdGVzcQB+ABNxAH4AGXEAfgAccQB+ACBzcQB+AAAAAAADdwQAAAACcQB+ADFxAH4AMnh0AANhZ2V+cQB+AAt0AAdJTlRFR0VSdAAFZW1haWxzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABG1hbGV+cQB+AAt0AAdCT09MRUFOeAB4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAfnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0Ac97CiAgIm9wIjogewogICAgIm5hbWUiOiAiRElWSURFIiwKICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDAsCiAgICAgICJuYW1lIjogIiQwIgogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAxMDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdLAogICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAibnVsbGFibGUiOiB0cnVlCiAgfSwKICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgImR5bmFtaWMiOiBmYWxzZQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAAHYmFsYW5jZX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABExPTkd4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml index 041d862fbc4..c54440bbf61 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml @@ -5,4 +5,4 @@ calcite: LogicalProject($f1=[+($3, 1)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"cnt":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBAXsKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDMsCiAgICAgICJuYW1lIjogIiQzIgogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAReHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABx4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB4AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABJ+cQB+AAt0AAZTVFJJTkd+cQB+ABh0AAdLZXl3b3JkcQB+AB14dAAHYWRkcmVzc3NxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgANdAAGZ2VuZGVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAEY2l0eXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGVtcGxveWVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAFc3RhdGVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AANhZ2VxAH4ADXQABWVtYWlsc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIbGFzdG5hbWVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h4AHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"cnt":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAfnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AQF7CiAgIm9wIjogewogICAgIm5hbWUiOiAiKyIsCiAgICAia2luZCI6ICJQTFVTIiwKICAgICJzeW50YXgiOiAiQklOQVJZIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiAwLAogICAgICAibmFtZSI6ICIkMCIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQAB2JhbGFuY2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05HeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest.json deleted file mode 100644 index 6afa7be030c..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(earliest_message=[$1], latest_message=[$2], server=[$0])\n LogicalAggregate(group=[{0}], earliest_message=[ARG_MIN($1, $2)], latest_message=[ARG_MAX($1, $2)])\n LogicalProject(server=[$1], message=[$3], @timestamp=[$2])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},earliest_message=ARG_MIN($1, $2),latest_message=ARG_MAX($1, $2)), PROJECT->[earliest_message, latest_message, server], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"server\":{\"terms\":{\"field\":\"server\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"earliest_message\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"message\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\":\"asc\"}}]}},\"latest_message\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"message\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\":\"desc\"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest.yaml new file mode 100644 index 00000000000..3dbd4ff8c20 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(earliest_message=[$1], latest_message=[$2], server=[$0]) + LogicalAggregate(group=[{0}], earliest_message=[ARG_MIN($1, $2)], latest_message=[ARG_MAX($1, $2)]) + LogicalProject(server=[$1], message=[$3], @timestamp=[$2]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},earliest_message=ARG_MIN($1, $2),latest_message=ARG_MAX($1, $2)), PROJECT->[earliest_message, latest_message, server], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"server":{"terms":{"field":"server","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"earliest_message":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"message.keyword"}],"sort":[{"@timestamp":{"order":"asc"}}]}},"latest_message":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"message.keyword"}],"sort":[{"@timestamp":{"order":"desc"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest_custom_time.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest_custom_time.json deleted file mode 100644 index 98d5277e419..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest_custom_time.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(earliest_message=[$1], latest_message=[$2], level=[$0])\n LogicalAggregate(group=[{0}], earliest_message=[ARG_MIN($1, $2)], latest_message=[ARG_MAX($1, $2)])\n LogicalProject(level=[$4], message=[$3], created_at=[$0])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},earliest_message=ARG_MIN($1, $2),latest_message=ARG_MAX($1, $2)), PROJECT->[earliest_message, latest_message, level], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"level\":{\"terms\":{\"field\":\"level\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"earliest_message\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"message\"],\"excludes\":[]},\"sort\":[{\"created_at\":{\"order\":\"asc\"}}]}},\"latest_message\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"message\"],\"excludes\":[]},\"sort\":[{\"created_at\":{\"order\":\"desc\"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest_custom_time.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest_custom_time.yaml new file mode 100644 index 00000000000..4dc24423d9d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest_custom_time.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(earliest_message=[$1], latest_message=[$2], level=[$0]) + LogicalAggregate(group=[{0}], earliest_message=[ARG_MIN($1, $2)], latest_message=[ARG_MAX($1, $2)]) + LogicalProject(level=[$4], message=[$3], created_at=[$0]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},earliest_message=ARG_MIN($1, $2),latest_message=ARG_MAX($1, $2)), PROJECT->[earliest_message, latest_message, level], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"level":{"terms":{"field":"level","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"earliest_message":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"message.keyword"}],"sort":[{"created_at":{"order":"asc"}}]}},"latest_message":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"message.keyword"}],"sort":[{"created_at":{"order":"desc"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_eventstats_dc.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_eventstats_dc.json index d9d477a31ea..4d7b5f07c6a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_eventstats_dc.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_eventstats_dc.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], distinct_states=[APPROX_DISTINCT_COUNT($7) OVER ()])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableWindow(window#0=[window(aggs [APPROX_DISTINCT_COUNT($7)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], distinct_states=[DISTINCT_COUNT_APPROX($7) OVER ()])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableWindow(window#0=[window(aggs [DISTINCT_COUNT_APPROX($7)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_eventstats_distinct_count.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_eventstats_distinct_count.json index c63ee04426f..6ffdb5d51eb 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_eventstats_distinct_count.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_eventstats_distinct_count.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], distinct_states=[APPROX_DISTINCT_COUNT($7) OVER (PARTITION BY $4)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableWindow(window#0=[window(partition {4} aggs [APPROX_DISTINCT_COUNT($7)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], distinct_states=[DISTINCT_COUNT_APPROX($7) OVER (PARTITION BY $4)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableWindow(window#0=[window(partition {4} aggs [DISTINCT_COUNT_APPROX($7)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_exists_correlated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_exists_correlated_subquery.yaml new file mode 100644 index 00000000000..920149399c0 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_exists_correlated_subquery.yaml @@ -0,0 +1,22 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(id=[$2], name=[$0], salary=[$4]) + LogicalSort(sort0=[$4], dir0=[DESC-nulls-last]) + LogicalFilter(condition=[EXISTS({ + LogicalProject(name=[$0], uid=[$1], occupation=[$2], department=[$3]) + LogicalFilter(condition=[=($cor0.id, $1)]) + LogicalSystemLimit(fetch=[10000], type=[SUBSEARCH_MAXOUT]) + LogicalFilter(condition=[=($0, 'Tom')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) + })], variablesSet=[[$cor0]]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + physical: | + EnumerableCalc(expr#0..3=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$2], dir0=[DESC-nulls-last]) + EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{1}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id, salary]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id","salary"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableAggregate(group=[{0}]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[true], expr#2=[$cor0], expr#3=[$t2.id], expr#4=[=($t3, $t0)], i=[$t1], $condition=[$t4]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[PROJECT->[name, uid], FILTER->=($0, 'Tom'), LIMIT->10000, PROJECT->[uid]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"term":{"name":{"value":"Tom","boost":1.0}}},"_source":{"includes":["uid"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_exists_uncorrelated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_exists_uncorrelated_subquery.yaml new file mode 100644 index 00000000000..c8c58c090b8 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_exists_uncorrelated_subquery.yaml @@ -0,0 +1,21 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(id=[$2], name=[$0], salary=[$4]) + LogicalSort(sort0=[$4], dir0=[DESC-nulls-last]) + LogicalFilter(condition=[EXISTS({ + LogicalSystemLimit(fetch=[10000], type=[SUBSEARCH_MAXOUT]) + LogicalProject(name=[$0], uid=[$1], occupation=[$2], department=[$3]) + LogicalFilter(condition=[=($0, 'Tom')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) + })], variablesSet=[[$cor0]]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) + EnumerableSort(sort0=[$2], dir0=[DESC-nulls-last]) + EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id, salary]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id","salary"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableAggregate(group=[{0}]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[true], i=[$t1]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[PROJECT->[name], FILTER->=($0, 'Tom'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"term":{"name":{"value":"Tom","boost":1.0}}},"_source":{"includes":["name"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_fillnull_value_syntax.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_fillnull_value_syntax.yaml new file mode 100644 index 00000000000..0f2ba239d73 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_fillnull_value_syntax.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[COALESCE($8, 0)], balance=[COALESCE($3, 0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[0], expr#3=[COALESCE($t0, $t2)], expr#4=[COALESCE($t1, $t2)], $f0=[$t3], $f1=[$t4]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age, balance], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_function_script_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_function_script_push.json deleted file mode 100644 index 101e68456c1..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_function_script_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(firstname=[$1], age=[$8])\n LogicalFilter(condition=[AND(=(CHAR_LENGTH($1), 5), =(ABS($8), 32), =($3, 39225))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, balance, age], SCRIPT->AND(=(CHAR_LENGTH($0), 5), =(ABS($2), 32), =($1, 39225)), PROJECT->[firstname, age], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"must\":[{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQBPnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJmaXJzdG5hbWUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0Aa17CiAgIm9wIjogewogICAgIm5hbWUiOiAiPSIsCiAgICAia2luZCI6ICJFUVVBTFMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAgICAgImtpbmQiOiAiQ0hBUl9MRU5HVEgiLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgIm5hbWUiOiAiJDAiCiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDUsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAN0AAlmaXJzdG5hbWVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AC3hwfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+ABF0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAYeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAaAAAAAHNxAH4AAAAAAAN3BAAAAAJ0AAdrZXl3b3Jkc3EAfgAMfnEAfgAQdAAGU1RSSU5HfnEAfgAUdAAHS2V5d29yZHEAfgAZeHQAB2JhbGFuY2V+cQB+ABB0AARMT05HdAADYWdlcQB+ACV4eA==\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQBPnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJmaXJzdG5hbWUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0Aal7CiAgIm9wIjogewogICAgIm5hbWUiOiAiPSIsCiAgICAia2luZCI6ICJFUVVBTFMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJBQlMiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMiwKICAgICAgICAgICJuYW1lIjogIiQyIgogICAgICAgIH0KICAgICAgXQogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAzMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAA3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgALeHB+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4AEXQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABh4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+ABoAAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+AAx+cQB+ABB0AAZTVFJJTkd+cQB+ABR0AAdLZXl3b3JkcQB+ABl4dAAHYmFsYW5jZX5xAH4AEHQABExPTkd0AANhZ2VxAH4AJXh4\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},{\"term\":{\"balance\":{\"value\":39225,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"firstname\",\"age\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_function_script_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_function_script_push.yaml new file mode 100644 index 00000000000..7d61dbc0b8b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_function_script_push.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(firstname=[$1], age=[$8]) + LogicalFilter(condition=[AND(=(CHAR_LENGTH($1), 5), =(ABS($8), 32), =($3, 39225))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, balance, age], SCRIPT->AND(=(CHAR_LENGTH($0), 5), =(ABS($2), 32), =($1, 39225)), PROJECT->[firstname, age], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAmHsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJmaXJzdG5hbWUiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQBrXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNIQVJfTEVOR1RIIiwKICAgICAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogNSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgALeHB+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4AEXQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABh4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+ABoAAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+AAx+cQB+ABB0AAZTVFJJTkd+cQB+ABR0AAdLZXl3b3JkcQB+ABl4eHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAensKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQBqXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkFCUyIsCiAgICAgICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgIm5hbWUiOiAiJDAiCiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDMyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3h4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"term":{"balance":{"value":39225,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname","age"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_timestamp_string.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_timestamp_string.yaml index ffb5fdff70c..e0a3fc8a7d3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_timestamp_string.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_timestamp_string.yaml @@ -6,4 +6,4 @@ calcite: LogicalFilter(condition=[>($3, TIMESTAMP('2016-12-08 00:00:00.000000000':VARCHAR))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], FILTER->SEARCH($3, Sarg[('2016-12-08 00:00:00':VARCHAR..'2018-11-09 00:00:00':VARCHAR)]:VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"range":{"birthdate":{"from":"2016-12-08T00:00:00.000Z","to":"2018-11-09T00:00:00.000Z","include_lower":false,"include_upper":false,"boost":1.0}}},"_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], FILTER->SEARCH($3, Sarg[('2016-12-08 00:00:00':VARCHAR..'2018-11-09 00:00:00':VARCHAR)]:VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"range":{"birthdate":{"from":"2016-12-08T00:00:00.000Z","to":"2018-11-09T00:00:00.000Z","include_lower":false,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.json deleted file mode 100644 index b6feda85160..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(firstname=[$1], age=[$8])\n LogicalFilter(condition=[AND(=($1, 'Amber'), =(-($8, 2), 30))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, age], SCRIPT->AND(=($0, 'Amber'), =(-($1, 2), 30)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"must\":[{\"term\":{\"firstname.keyword\":{\"value\":\"Amber\",\"boost\":1.0}}},{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQA6XsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJmaXJzdG5hbWUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQCcnsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIi0iLAogICAgICAgICJraW5kIjogIk1JTlVTIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDEsCiAgICAgICAgICAibmFtZSI6ICIkMSIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogMiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDMwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAACdAAJZmlyc3RuYW1lc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4ADH5xAH4AEHQABlNUUklOR35xAH4AFHQAB0tleXdvcmRxAH4AGXh0AANhZ2V+cQB+ABB0AARMT05HeHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"firstname\",\"age\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.yaml new file mode 100644 index 00000000000..4992957b230 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(firstname=[$1], age=[$8]) + LogicalFilter(condition=[AND(=($1, 'Amber'), =(-($8, 2), 30))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, age], SCRIPT->AND(=($0, 'Amber'), =(-($1, 2), 30)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"term":{"firstname.keyword":{"value":"Amber","boost":1.0}}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAensKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQCcnsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIi0iLAogICAgICAgICJraW5kIjogIk1JTlVTIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogMiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDMwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3h4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname","age"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_with_search.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_with_search.yaml index ceb491f9766..29ebac7168f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_with_search.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_with_search.yaml @@ -8,4 +8,4 @@ calcite: LogicalFilter(condition=[AND(>=($3, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <($3, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[birthdate], FILTER->SEARCH($0, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR); NULL AS FALSE]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1d)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"range":{"birthdate":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"boost":1.0}}},{"exists":{"field":"birthdate","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["birthdate"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(birthdate,1d)":{"date_histogram":{"field":"birthdate","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[birthdate], FILTER->SEARCH($0, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR); NULL AS FALSE]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1d)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"range":{"birthdate":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"exists":{"field":"birthdate","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["birthdate"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(birthdate,1d)":{"date_histogram":{"field":"birthdate","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_first_last.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_first_last.json deleted file mode 100644 index d10259db2ef..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_first_last.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(first_name=[$1], last_name=[$2], gender=[$0])\n LogicalAggregate(group=[{0}], first_name=[FIRST($1)], last_name=[LAST($1)])\n LogicalProject(gender=[$4], firstname=[$1])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},first_name=FIRST($1),last_name=LAST($1)), PROJECT->[first_name, last_name, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"first_name\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]}}},\"last_name\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"desc\"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_first_last.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_first_last.yaml new file mode 100644 index 00000000000..30f505a24a7 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_first_last.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(first_name=[$1], last_name=[$2], gender=[$0]) + LogicalAggregate(group=[{0}], first_name=[FIRST($1)], last_name=[LAST($1)]) + LogicalProject(gender=[$4], firstname=[$1]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},first_name=FIRST($1),last_name=LAST($1)), PROJECT->[first_name, last_name, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"first_name":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"firstname"}]}},"last_name":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"firstname"}],"sort":[{"_doc":{"order":"desc"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_in_correlated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_in_correlated_subquery.yaml new file mode 100644 index 00000000000..7c9b83b75f5 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_in_correlated_subquery.yaml @@ -0,0 +1,23 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(id=[$2], name=[$0], salary=[$4]) + LogicalSort(sort0=[$4], dir0=[DESC-nulls-last]) + LogicalFilter(condition=[IN($0, { + LogicalProject(name=[$0]) + LogicalFilter(condition=[=($cor0.id, $1)]) + LogicalSystemLimit(fetch=[10000], type=[SUBSEARCH_MAXOUT]) + LogicalFilter(condition=[=($0, 'Tom')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) + })], variablesSet=[[$cor0]]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + physical: | + EnumerableCalc(expr#0..3=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$2], dir0=[DESC-nulls-last]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[=($t0, $t3)], proj#0..3=[{exprs}], $condition=[$t4]) + EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{1}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id, salary]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id","salary"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableAggregate(group=[{0}]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[$cor0], expr#3=[$t2.id], expr#4=[=($t3, $t1)], proj#0..1=[{exprs}], $condition=[$t4]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[PROJECT->[name, uid], FILTER->=($0, 'Tom'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"term":{"name":{"value":"Tom","boost":1.0}}},"_source":{"includes":["name","uid"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_in_uncorrelated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_in_uncorrelated_subquery.yaml new file mode 100644 index 00000000000..42de0ae2f46 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_in_uncorrelated_subquery.yaml @@ -0,0 +1,18 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(id=[$2], name=[$0], salary=[$4]) + LogicalSort(sort0=[$4], dir0=[DESC-nulls-last]) + LogicalFilter(condition=[IN($2, { + LogicalSystemLimit(fetch=[10000], type=[SUBSEARCH_MAXOUT]) + LogicalProject(uid=[$1]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) + })], variablesSet=[[$cor0]]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) + EnumerableSort(sort0=[$2], dir0=[DESC-nulls-last]) + EnumerableHashJoin(condition=[=($1, $3)], joinType=[semi]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id, salary]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id","salary"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[PROJECT->[uid], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["uid"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_isblank.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_isblank.json deleted file mode 100644 index 876f6f6972e..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_isblank.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10])\n LogicalFilter(condition=[OR(IS NULL($1), IS EMPTY(TRIM(FLAG(BOTH), ' ', $1)))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->OR(IS NULL($1), IS EMPTY(TRIM(FLAG(BOTH), ' ', $1))), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQEj3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BDR7CiAgIm9wIjogewogICAgIm5hbWUiOiAiT1IiLAogICAgImtpbmQiOiAiT1IiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJJUyBOVUxMIiwKICAgICAgICAia2luZCI6ICJJU19OVUxMIiwKICAgICAgICAic3ludGF4IjogIlBPU1RGSVgiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiaW5wdXQiOiAxLAogICAgICAgICAgIm5hbWUiOiAiJDEiCiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiSVMgRU1QVFkiLAogICAgICAgICJraW5kIjogIk9USEVSIiwKICAgICAgICAic3ludGF4IjogIlBPU1RGSVgiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAib3AiOiB7CiAgICAgICAgICAgICJuYW1lIjogIlRSSU0iLAogICAgICAgICAgICAia2luZCI6ICJUUklNIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6ICJCT1RIIiwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIlNZTUJPTCIsCiAgICAgICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICAgICAgICAgIH0KICAgICAgICAgICAgfSwKICAgICAgICAgICAgewogICAgICAgICAgICAgICJsaXRlcmFsIjogIiAiLAogICAgICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAiQ0hBUiIsCiAgICAgICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAgICAgICAgICJwcmVjaXNpb24iOiAxCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImlucHV0IjogMSwKICAgICAgICAgICAgICAibmFtZSI6ICIkMSIKICAgICAgICAgICAgfQogICAgICAgICAgXQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAQeHB+cQB+AAp0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4AC3QABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABt4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB0AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABF+cQB+AAp0AAZTVFJJTkd+cQB+ABd0AAdLZXl3b3JkcQB+ABx4dAAHYWRkcmVzc3NxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgAMdAAGZ2VuZGVyc3EAfgAPcQB+ABVxAH4AGHEAfgAcc3EAfgAAAAAAA3cEAAAAAnEAfgAhcQB+ACJ4dAAEY2l0eXNxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAJxAH4AIXEAfgAieHQACGVtcGxveWVyc3EAfgAPcQB+ABVxAH4AGHEAfgAcc3EAfgAAAAAAA3cEAAAAAnEAfgAhcQB+ACJ4dAAFc3RhdGVzcQB+AA9xAH4AFXEAfgAYcQB+ABxzcQB+AAAAAAADdwQAAAACcQB+ACFxAH4AInh0AANhZ2VxAH4ADHQABWVtYWlsc3EAfgAPcQB+ABVxAH4AGHEAfgAcc3EAfgAAAAAAA3cEAAAAAnEAfgAhcQB+ACJ4dAAIbGFzdG5hbWVzcQB+AA9xAH4AFXEAfgAYcQB+ABxzcQB+AAAAAAADdwQAAAACcQB+ACFxAH4AInh4eA==\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_isblank.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_isblank.yaml new file mode 100644 index 00000000000..aa739610250 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_isblank.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalFilter(condition=[OR(IS NULL($1), IS EMPTY(TRIM(FLAG(BOTH), ' ', $1)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->OR(IS NULL($1), IS EMPTY(TRIM(FLAG(BOTH), ' ', $1))), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAmHsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJmaXJzdG5hbWUiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQENHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJPUiIsCiAgICAia2luZCI6ICJPUiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIklTIE5VTEwiLAogICAgICAgICJraW5kIjogIklTX05VTEwiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJJUyBFTVBUWSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVIiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiVFJJTSIsCiAgICAgICAgICAgICJraW5kIjogIlRSSU0iLAogICAgICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICAgICAgfSwKICAgICAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICAgICAgewogICAgICAgICAgICAgICJsaXRlcmFsIjogIkJPVEgiLAogICAgICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAiU1lNQk9MIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAiICIsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJDSEFSIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAgICAgInByZWNpc2lvbiI6IDEKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9CiAgICAgICAgICBdCiAgICAgICAgfQogICAgICBdCiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAAJZmlyc3RuYW1lc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4ADH5xAH4AEHQABlNUUklOR35xAH4AFHQAB0tleXdvcmRxAH4AGXh4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty.json deleted file mode 100644 index 72c518aaab1..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10])\n LogicalFilter(condition=[OR(IS NULL($1), IS EMPTY($1))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->OR(IS NULL($1), IS EMPTY($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQEj3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AgV7CiAgIm9wIjogewogICAgIm5hbWUiOiAiT1IiLAogICAgImtpbmQiOiAiT1IiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJJUyBOVUxMIiwKICAgICAgICAia2luZCI6ICJJU19OVUxMIiwKICAgICAgICAic3ludGF4IjogIlBPU1RGSVgiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiaW5wdXQiOiAxLAogICAgICAgICAgIm5hbWUiOiAiJDEiCiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiSVMgRU1QVFkiLAogICAgICAgICJraW5kIjogIk9USEVSIiwKICAgICAgICAic3ludGF4IjogIlBPU1RGSVgiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiaW5wdXQiOiAxLAogICAgICAgICAgIm5hbWUiOiAiJDEiCiAgICAgICAgfQogICAgICBdCiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAALdAAOYWNjb3VudF9udW1iZXJ+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05HdAAJZmlyc3RuYW1lc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+ABB4cH5xAH4ACnQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgALdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AG3hwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AHQAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AEX5xAH4ACnQABlNUUklOR35xAH4AF3QAB0tleXdvcmRxAH4AHHh0AAdhZGRyZXNzc3EAfgAPcQB+ABVxAH4AGHEAfgAcc3EAfgAAAAAAA3cEAAAAAHh0AAdiYWxhbmNlcQB+AAx0AAZnZW5kZXJzcQB+AA9xAH4AFXEAfgAYcQB+ABxzcQB+AAAAAAADdwQAAAACcQB+ACFxAH4AInh0AARjaXR5c3EAfgAPcQB+ABVxAH4AGHEAfgAcc3EAfgAAAAAAA3cEAAAAAnEAfgAhcQB+ACJ4dAAIZW1wbG95ZXJzcQB+AA9xAH4AFXEAfgAYcQB+ABxzcQB+AAAAAAADdwQAAAACcQB+ACFxAH4AInh0AAVzdGF0ZXNxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAJxAH4AIXEAfgAieHQAA2FnZXEAfgAMdAAFZW1haWxzcQB+AA9xAH4AFXEAfgAYcQB+ABxzcQB+AAAAAAADdwQAAAACcQB+ACFxAH4AInh0AAhsYXN0bmFtZXNxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAJxAH4AIXEAfgAieHh4\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty.yaml new file mode 100644 index 00000000000..349be65b454 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalFilter(condition=[OR(IS NULL($1), IS EMPTY($1))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->OR(IS NULL($1), IS EMPTY($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAmHsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJmaXJzdG5hbWUiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQCBXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJPUiIsCiAgICAia2luZCI6ICJPUiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIklTIE5VTEwiLAogICAgICAgICJraW5kIjogIklTX05VTEwiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJJUyBFTVBUWSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVIiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AAlmaXJzdG5hbWVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AC3hwfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+ABF0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAYeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAaAAAAAHNxAH4AAAAAAAN3BAAAAAJ0AAdrZXl3b3Jkc3EAfgAMfnEAfgAQdAAGU1RSSU5HfnEAfgAUdAAHS2V5d29yZHEAfgAZeHh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty_or_others.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty_or_others.json deleted file mode 100644 index 7d7bcbbf4e6..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty_or_others.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10])\n LogicalFilter(condition=[OR(=($4, 'M'), IS NULL($1), IS EMPTY($1))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->OR(IS NULL($1), =($4, 'M'), IS EMPTY($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQEj3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0A3V7CiAgIm9wIjogewogICAgIm5hbWUiOiAiT1IiLAogICAgImtpbmQiOiAiT1IiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJJUyBOVUxMIiwKICAgICAgICAia2luZCI6ICJJU19OVUxMIiwKICAgICAgICAic3ludGF4IjogIlBPU1RGSVgiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiaW5wdXQiOiAxLAogICAgICAgICAgIm5hbWUiOiAiJDEiCiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiPSIsCiAgICAgICAgImtpbmQiOiAiRVFVQUxTIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDQsCiAgICAgICAgICAibmFtZSI6ICIkNCIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogIk0iLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IDEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJJUyBFTVBUWSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVIiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDEsCiAgICAgICAgICAibmFtZSI6ICIkMSIKICAgICAgICB9CiAgICAgIF0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAt0AA5hY2NvdW50X251bWJlcn5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABExPTkd0AAlmaXJzdG5hbWVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AEHhwfnEAfgAKdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AAt0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAbeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAdAAAAAHNxAH4AAAAAAAN3BAAAAAJ0AAdrZXl3b3Jkc3EAfgARfnEAfgAKdAAGU1RSSU5HfnEAfgAXdAAHS2V5d29yZHEAfgAceHQAB2FkZHJlc3NzcQB+AA9xAH4AFXEAfgAYcQB+ABxzcQB+AAAAAAADdwQAAAAAeHQAB2JhbGFuY2VxAH4ADHQABmdlbmRlcnNxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAJxAH4AIXEAfgAieHQABGNpdHlzcQB+AA9xAH4AFXEAfgAYcQB+ABxzcQB+AAAAAAADdwQAAAACcQB+ACFxAH4AInh0AAhlbXBsb3llcnNxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAJxAH4AIXEAfgAieHQABXN0YXRlc3EAfgAPcQB+ABVxAH4AGHEAfgAcc3EAfgAAAAAAA3cEAAAAAnEAfgAhcQB+ACJ4dAADYWdlcQB+AAx0AAVlbWFpbHNxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAJxAH4AIXEAfgAieHQACGxhc3RuYW1lc3EAfgAPcQB+ABVxAH4AGHEAfgAcc3EAfgAAAAAAA3cEAAAAAnEAfgAhcQB+ACJ4eHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty_or_others.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty_or_others.yaml new file mode 100644 index 00000000000..b940a11198e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty_or_others.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalFilter(condition=[OR(=($4, 'M'), IS NULL($1), IS EMPTY($1))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->OR(IS NULL($1), =($4, 'M'), IS EMPTY($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQBBHsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJmaXJzdG5hbWUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJnZW5kZXIiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQDdXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJPUiIsCiAgICAia2luZCI6ICJPUiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIklTIE5VTEwiLAogICAgICAgICJraW5kIjogIklTX05VTEwiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICI9IiwKICAgICAgICAia2luZCI6ICJFUVVBTFMiLAogICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMSwKICAgICAgICAgICJuYW1lIjogIiQxIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAiTSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfSwKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIklTIEVNUFRZIiwKICAgICAgICAia2luZCI6ICJPVEhFUiIsCiAgICAgICAgInN5bnRheCI6ICJQT1NURklYIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAnQACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgALeHB+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4AEXQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABh4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+ABoAAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+AAx+cQB+ABB0AAZTVFJJTkd+cQB+ABR0AAdLZXl3b3JkcQB+ABl4dAAGZ2VuZGVyc3EAfgAKcQB+ABJxAH4AFXEAfgAZc3EAfgAAAAAAA3cEAAAAAnEAfgAecQB+AB94eHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_agg.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_agg.yaml new file mode 100644 index 00000000000..f59f795a35a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_agg.yaml @@ -0,0 +1,18 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(COUNT()=[$0], age=[$1], gender=[$2], overall_cnt=[$3], R.gender=[$4]) + LogicalJoin(condition=[=($2, $4)], joinType=[inner]) + LogicalProject(COUNT()=[$2], age=[$0], gender=[$1]) + LogicalAggregate(group=[{0, 1}], COUNT()=[COUNT()]) + LogicalProject(age=[$8], gender=[$4]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(overall_cnt=[$1], gender=[$0]) + LogicalAggregate(group=[{0}], overall_cnt=[COUNT()]) + LogicalProject(gender=[$4]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableMergeJoin(condition=[=($2, $4)], joinType=[inner]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},COUNT()=COUNT()), PROJECT->[COUNT(), age, gender], SORT->[2]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},overall_cnt=COUNT()), PROJECT->[overall_cnt, gender], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_fields.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_fields.json deleted file mode 100644 index 0a662c047ee..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_fields.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$13], firstname=[$14], address=[$15], birthdate=[$16], gender=[$17], city=[$18], lastname=[$19], balance=[$20], employer=[$21], state=[$22], age=[$23], email=[$24], male=[$25])\n LogicalJoin(condition=[=($0, $13)], joinType=[left])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableCalc(expr#0..13=[{inputs}], account_number=[$t1], firstname=[$t2], address=[$t3], birthdate=[$t4], gender=[$t5], city=[$t6], lastname=[$t7], balance=[$t8], employer=[$t9], state=[$t10], age=[$t11], email=[$t12], male=[$t13])\n EnumerableLimit(fetch=[10000])\n EnumerableMergeJoin(condition=[=($0, $1)], joinType=[left])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number], SORT->[{\n \"account_number\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_last\"\n }\n}]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\"],\"excludes\":[]},\"sort\":[{\"account_number\":{\"order\":\"asc\",\"missing\":\"_last\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT->[{\n \"account_number\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_last\"\n }\n}]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"birthdate\",\"gender\",\"city\",\"lastname\",\"balance\",\"employer\",\"state\",\"age\",\"email\",\"male\"],\"excludes\":[]},\"sort\":[{\"account_number\":{\"order\":\"asc\",\"missing\":\"_last\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_fields.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_fields.yaml new file mode 100644 index 00000000000..12259a8e5ae --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_join_with_fields.yaml @@ -0,0 +1,26 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$13], firstname=[$14], address=[$15], birthdate=[$16], gender=[$17], city=[$18], lastname=[$19], balance=[$20], employer=[$21], state=[$22], age=[$23], email=[$24], male=[$25]) + LogicalJoin(condition=[=($0, $13)], joinType=[left]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableCalc(expr#0..13=[{inputs}], account_number=[$t1], firstname=[$t2], address=[$t3], birthdate=[$t4], gender=[$t5], city=[$t6], lastname=[$t7], balance=[$t8], employer=[$t9], state=[$t10], age=[$t11], email=[$t12], male=[$t13]) + EnumerableLimit(fetch=[10000]) + EnumerableMergeJoin(condition=[=($0, $1)], joinType=[left]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number], LIMIT->10000, SORT->[{ + "account_number" : { + "order" : "asc", + "missing" : "_last" + } + }]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number"],"excludes":[]},"sort":[{"account_number":{"order":"asc","missing":"_last"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], LIMIT->50000, SORT->[{ + "account_number" : { + "order" : "asc", + "missing" : "_last" + } + }]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":50000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]},"sort":[{"account_number":{"order":"asc","missing":"_last"}}]}, requestedTotalSize=50000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_like_function.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_like_function.json deleted file mode 100644 index 2be777fddee..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_like_function.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10])\n LogicalFilter(condition=[ILIKE($1, '%mbe%':VARCHAR, '\\')])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->ILIKE($1, '%mbe%':VARCHAR, '\\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"wildcard\":{\"firstname.keyword\":{\"wildcard\":\"*mbe*\",\"case_insensitive\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_like_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_like_function.yaml new file mode 100644 index 00000000000..3a891dc6bc4 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_like_function.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalFilter(condition=[ILIKE($1, '%mbe%', '\')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->ILIKE($1, '%mbe%', '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"wildcard":{"firstname.keyword":{"wildcard":"*mbe*","case_insensitive":true,"boost":1.0}}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown4.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown4.json deleted file mode 100644 index 635a33b5d6e..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown4.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(offset=[10], fetch=[10])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], fetch=[100])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n EnumerableLimit(offset=[10], fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT->[0 ASC FIRST], LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":20,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown4.yaml new file mode 100644 index 00000000000..d13ed6aa8d3 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown4.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(offset=[10], fetch=[10]) + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], fetch=[100]) + LogicalProject(count()=[$1], state=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(state=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableLimit(offset=[10], fetch=[10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state], SORT->[1 ASC FIRST], LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":20,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable1.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable1.json deleted file mode 100644 index c4346b4134b..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable1.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(offset=[10], fetch=[10])\n LogicalSort(fetch=[100])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n LogicalFilter(condition=[IS NOT NULL($7)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n EnumerableLimit(offset=[10], fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($7), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"state\",\"boost\":1.0}},\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":20,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":false,\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable1.yaml new file mode 100644 index 00000000000..b4117a5a84c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable1.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(offset=[10], fetch=[10]) + LogicalSort(fetch=[100]) + LogicalProject(count()=[$1], state=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(state=[$7]) + LogicalFilter(condition=[IS NOT NULL($7)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0]) + EnumerableLimit(offset=[10], fetch=[10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":20,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable2.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable2.json deleted file mode 100644 index e391db7e53e..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable2.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(offset=[10], fetch=[10])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], fetch=[100])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n LogicalFilter(condition=[IS NOT NULL($7)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n EnumerableLimit(offset=[10], fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($7), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT->[0 ASC FIRST], LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"state\",\"boost\":1.0}},\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":20,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":false,\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable2.yaml new file mode 100644 index 00000000000..46e8cab736b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable2.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(offset=[10], fetch=[10]) + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], fetch=[100]) + LogicalProject(count()=[$1], state=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(state=[$7]) + LogicalFilter(condition=[IS NOT NULL($7)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableLimit(offset=[10], fetch=[10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state], SORT->[1 ASC FIRST], LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":20,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_sort_push.yaml index 1fdacd0c05b..7ab0399e85f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_sort_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_sort_push.yaml @@ -6,6 +6,9 @@ calcite: LogicalSort(fetch=[5]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableLimit(fetch=[10000]) - EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->5, SORT->[{ + "age" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]},"sort":[{"age":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_max_string_field.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_max_string_field.json deleted file mode 100644 index 961cdc4687f..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_max_string_field.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], max(firstname)=[MAX($0)])\n LogicalProject(firstname=[$1])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},max(firstname)=MAX($0))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"max(firstname)\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]},\"sort\":[{\"firstname.keyword\":{\"order\":\"desc\"}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_max_string_field.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_max_string_field.yaml new file mode 100644 index 00000000000..9f44afdf68d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_max_string_field.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], max(firstname)=[MAX($0)]) + LogicalProject(firstname=[$1]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},max(firstname)=MAX($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"max(firstname)":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"firstname.keyword"}],"sort":[{"firstname.keyword":{"order":"desc"}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_merge_join_sort_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_merge_join_sort_push.json deleted file mode 100644 index 19bfbbaa6b9..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_merge_join_sort_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], r.account_number=[$13], r.firstname=[$14], r.address=[$15], r.birthdate=[$16], r.gender=[$17], r.city=[$18], r.lastname=[$19], r.balance=[$20], r.employer=[$21], r.state=[$22], r.age=[$23], r.email=[$24], r.male=[$25])\n LogicalJoin(condition=[=($0, $13)], joinType=[inner])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableMergeJoin(condition=[=($0, $13)], joinType=[inner])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT->[{\n \"account_number\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_last\"\n }\n}]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"birthdate\",\"gender\",\"city\",\"lastname\",\"balance\",\"employer\",\"state\",\"age\",\"email\",\"male\"],\"excludes\":[]},\"sort\":[{\"account_number\":{\"order\":\"asc\",\"missing\":\"_last\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT->[{\n \"account_number\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_last\"\n }\n}]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"birthdate\",\"gender\",\"city\",\"lastname\",\"balance\",\"employer\",\"state\",\"age\",\"email\",\"male\"],\"excludes\":[]},\"sort\":[{\"account_number\":{\"order\":\"asc\",\"missing\":\"_last\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_merge_join_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_merge_join_sort_push.yaml new file mode 100644 index 00000000000..7c5f95fba77 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_merge_join_sort_push.yaml @@ -0,0 +1,25 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], r.account_number=[$13], r.firstname=[$14], r.address=[$15], r.birthdate=[$16], r.gender=[$17], r.city=[$18], r.lastname=[$19], r.balance=[$20], r.employer=[$21], r.state=[$22], r.age=[$23], r.email=[$24], r.male=[$25]) + LogicalJoin(condition=[=($0, $13)], joinType=[inner]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableMergeJoin(condition=[=($0, $13)], joinType=[inner]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT->[{ + "account_number" : { + "order" : "asc", + "missing" : "_last" + } + }]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]},"sort":[{"account_number":{"order":"asc","missing":"_last"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], LIMIT->50000, SORT->[{ + "account_number" : { + "order" : "asc", + "missing" : "_last" + } + }]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":50000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]},"sort":[{"account_number":{"order":"asc","missing":"_last"}}]}, requestedTotalSize=50000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_min_max_agg_on_derived_field.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_min_max_agg_on_derived_field.yaml new file mode 100644 index 00000000000..805c42527a8 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_min_max_agg_on_derived_field.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], MIN(balance2)=[MIN($0)], MAX(balance2)=[MAX($0)]) + LogicalProject(balance2=[CEIL(DIVIDE($3, 10000.0:DECIMAL(6, 1)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},MIN(balance2)=MIN($0),MAX(balance2)=MAX($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"MIN(balance2)":{"min":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAfnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0Awl7CiAgIm9wIjogewogICAgIm5hbWUiOiAiQ0VJTCIsCiAgICAia2luZCI6ICJDRUlMIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJESVZJREUiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAxMDAwMC4wLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkRFQ0lNQUwiLAogICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IDYsCiAgICAgICAgICAgICJzY2FsZSI6IDEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiREVDSU1BTCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogMjcsCiAgICAgICAgInNjYWxlIjogNwogICAgICB9LAogICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICJkeW5hbWljIjogZmFsc2UKICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AAdiYWxhbmNlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3h4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}},"MAX(balance2)":{"max":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAfnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0Awl7CiAgIm9wIjogewogICAgIm5hbWUiOiAiQ0VJTCIsCiAgICAia2luZCI6ICJDRUlMIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJESVZJREUiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAxMDAwMC4wLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkRFQ0lNQUwiLAogICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IDYsCiAgICAgICAgICAgICJzY2FsZSI6IDEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiREVDSU1BTCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogMjcsCiAgICAgICAgInNjYWxlIjogNwogICAgICB9LAogICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICJkeW5hbWljIjogZmFsc2UKICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AAdiYWxhbmNlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3h4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_min_string_field.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_min_string_field.json deleted file mode 100644 index 41a14f5e84e..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_min_string_field.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], min(firstname)=[MIN($0)])\n LogicalProject(firstname=[$1])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},min(firstname)=MIN($0))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"min(firstname)\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]},\"sort\":[{\"firstname.keyword\":{\"order\":\"asc\"}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_min_string_field.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_min_string_field.yaml new file mode 100644 index 00000000000..20bfd6b4dd7 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_min_string_field.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], min(firstname)=[MIN($0)]) + LogicalProject(firstname=[$1]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},min(firstname)=MIN($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"min(firstname)":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"firstname.keyword"}],"sort":[{"firstname.keyword":{"order":"asc"}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml new file mode 100644 index 00000000000..6a5bc8ea0f5 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC-nulls-first]) + LogicalProject(c=[$1], s=[$2], state=[$0]) + LogicalAggregate(group=[{0}], c=[COUNT()], s=[SUM($1)]) + LogicalProject(state=[$7], balance=[$3]) + LogicalFilter(condition=[IS NOT NULL($7)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),s=SUM($0)), PROJECT->[c, s, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml new file mode 100644 index 00000000000..d1651f464a6 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) + LogicalProject(c=[$1], s=[$2], state=[$0]) + LogicalAggregate(group=[{0}], c=[COUNT()], s=[SUM($1)]) + LogicalProject(state=[$7], balance=[$3]) + LogicalFilter(condition=[IS NOT NULL($7)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),s=SUM($0)), PROJECT->[c, s, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_multisearch_basic.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_multisearch_basic.yaml new file mode 100644 index 00000000000..8fe5241ced4 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_multisearch_basic.yaml @@ -0,0 +1,22 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count=[$1], age_group=[$0]) + LogicalAggregate(group=[{0}], count=[COUNT()]) + LogicalProject(age_group=[$11]) + LogicalUnion(all=[true]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], age_group=['young':VARCHAR]) + LogicalFilter(condition=[<($8, 30)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], age_group=['adult':VARCHAR]) + LogicalFilter(condition=[>=($8, 30)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], count=[$t1], age_group=[$t0]) + EnumerableAggregate(group=[{0}], count=[COUNT()]) + EnumerableUnion(all=[true]) + EnumerableCalc(expr#0=[{inputs}], expr#1=['young':VARCHAR], age_group=[$t1]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], FILTER-><($0, 30)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"range":{"age":{"from":null,"to":30,"include_lower":true,"include_upper":false,"boost":1.0}}},"_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0=[{inputs}], expr#1=['adult':VARCHAR], age_group=[$t1]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], FILTER->>=($0, 30)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":true,"include_upper":true,"boost":1.0}}},"_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_multisearch_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_multisearch_timestamp.yaml new file mode 100644 index 00000000000..b38889b0323 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_multisearch_timestamp.yaml @@ -0,0 +1,27 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC], fetch=[5]) + LogicalUnion(all=[true]) + LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) + LogicalFilter(condition=[SEARCH($1, Sarg['A':VARCHAR, 'B':VARCHAR]:VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) + LogicalFilter(condition=[SEARCH($1, Sarg['E':VARCHAR, 'F':VARCHAR]:VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data2]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableLimit(fetch=[5]) + EnumerableMergeUnion(all=[true]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[@timestamp, category, value, timestamp], FILTER->SEARCH($1, Sarg['A':VARCHAR, 'B':VARCHAR]:VARCHAR), SORT->[{ + "@timestamp" : { + "order" : "desc", + "missing" : "_first" + } + }], LIMIT->5, LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","query":{"terms":{"category":["A","B"],"boost":1.0}},"_source":{"includes":["@timestamp","category","value","timestamp"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_first"}}]}, requestedTotalSize=5, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data2]], PushDownContext=[[PROJECT->[@timestamp, category, value, timestamp], FILTER->SEARCH($1, Sarg['E':VARCHAR, 'F':VARCHAR]:VARCHAR), SORT->[{ + "@timestamp" : { + "order" : "desc", + "missing" : "_first" + } + }], LIMIT->5, LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","query":{"terms":{"category":["E","F"],"boost":1.0}},"_source":{"includes":["@timestamp","category","value","timestamp"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_first"}}]}, requestedTotalSize=5, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.yaml index fe1413c12e8..16c87b3ae02 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0..2=[{inputs}], expr#3=[PATTERN_PARSER($t0, $t2)], expr#4=['pattern'], expr#5=[ITEM($t3, $t4)], expr#6=[SAFE_CAST($t5)], expr#7=['tokens'], expr#8=[ITEM($t3, $t7)], expr#9=[SAFE_CAST($t8)], patterns_field=[$t6], pattern_count=[$t1], tokens=[$t9], sample_logs=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},pattern_count=COUNT($1),sample_logs=TAKE($0, $2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"patterns_field":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQGGXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0VBUkNIIiwKICAgICAgICAia2luZCI6ICJTRUFSQ0giLAogICAgICAgICJzeW50YXgiOiAiSU5URVJOQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiaW5wdXQiOiA5LAogICAgICAgICAgIm5hbWUiOiAiJDkiCiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAibGl0ZXJhbCI6IHsKICAgICAgICAgICAgInJhbmdlU2V0IjogWwogICAgICAgICAgICAgIFsKICAgICAgICAgICAgICAgICJzaW5nbGV0b24iLAogICAgICAgICAgICAgICAgIntcInZhbHVlXCI6XCJcIixcImNoYXJzZXROYW1lXCI6XCJJU08tODg1OS0xXCIsXCJjb2xsYXRpb25cIjp7XCJjb2xsYXRpb25OYW1lXCI6XCJJU08tODg1OS0xJGVuX1VTJHByaW1hcnlcIixcImNvZXJjaWJpbGl0eVwiOlwiSU1QTElDSVRcIixcImxvY2FsZVwiOlwiZW5fVVNcIn0sXCJjaGFyc2V0XCI6XCJJU08tODg1OS0xXCIsXCJ2YWx1ZUJ5dGVzXCI6bnVsbH0iCiAgICAgICAgICAgICAgXQogICAgICAgICAgICBdLAogICAgICAgICAgICAibnVsbEFzIjogIlRSVUUiCiAgICAgICAgICB9LAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICIiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIlJFR0VYUF9SRVBMQUNFIiwKICAgICAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDksCiAgICAgICAgICAibmFtZSI6ICIkOSIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogIlthLXpBLVowLTldKyIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogIjwqPiIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAF2phdmEudXRpbC5MaW5rZWRIYXNoTWFwNMBOXBBswPsCAAFaAAthY2Nlc3NPcmRlcnhyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAALdAAOYWNjb3VudF9udW1iZXJ+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05HdAAJZmlyc3RuYW1lc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+ABF4cH5xAH4AC3QAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgAMdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AHHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AHgAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AEn5xAH4AC3QABlNUUklOR35xAH4AGHQAB0tleXdvcmRxAH4AHXh0AAdhZGRyZXNzc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAHh0AAdiYWxhbmNlcQB+AA10AAZnZW5kZXJzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AARjaXR5c3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIZW1wbG95ZXJzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AAVzdGF0ZXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQAA2FnZXEAfgANdAAFZW1haWxzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AAhsYXN0bmFtZXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHgAeA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"pattern_count":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQGGXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0VBUkNIIiwKICAgICAgICAia2luZCI6ICJTRUFSQ0giLAogICAgICAgICJzeW50YXgiOiAiSU5URVJOQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiaW5wdXQiOiA5LAogICAgICAgICAgIm5hbWUiOiAiJDkiCiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAibGl0ZXJhbCI6IHsKICAgICAgICAgICAgInJhbmdlU2V0IjogWwogICAgICAgICAgICAgIFsKICAgICAgICAgICAgICAgICJzaW5nbGV0b24iLAogICAgICAgICAgICAgICAgIntcInZhbHVlXCI6XCJcIixcImNoYXJzZXROYW1lXCI6XCJJU08tODg1OS0xXCIsXCJjb2xsYXRpb25cIjp7XCJjb2xsYXRpb25OYW1lXCI6XCJJU08tODg1OS0xJGVuX1VTJHByaW1hcnlcIixcImNvZXJjaWJpbGl0eVwiOlwiSU1QTElDSVRcIixcImxvY2FsZVwiOlwiZW5fVVNcIn0sXCJjaGFyc2V0XCI6XCJJU08tODg1OS0xXCIsXCJ2YWx1ZUJ5dGVzXCI6bnVsbH0iCiAgICAgICAgICAgICAgXQogICAgICAgICAgICBdLAogICAgICAgICAgICAibnVsbEFzIjogIlRSVUUiCiAgICAgICAgICB9LAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICIiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIlJFR0VYUF9SRVBMQUNFIiwKICAgICAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDksCiAgICAgICAgICAibmFtZSI6ICIkOSIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogIlthLXpBLVowLTldKyIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogIjwqPiIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAF2phdmEudXRpbC5MaW5rZWRIYXNoTWFwNMBOXBBswPsCAAFaAAthY2Nlc3NPcmRlcnhyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAALdAAOYWNjb3VudF9udW1iZXJ+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05HdAAJZmlyc3RuYW1lc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+ABF4cH5xAH4AC3QAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgAMdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AHHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AHgAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AEn5xAH4AC3QABlNUUklOR35xAH4AGHQAB0tleXdvcmRxAH4AHXh0AAdhZGRyZXNzc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAHh0AAdiYWxhbmNlcQB+AA10AAZnZW5kZXJzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AARjaXR5c3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIZW1wbG95ZXJzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AAVzdGF0ZXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQAA2FnZXEAfgANdAAFZW1haWxzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AAhsYXN0bmFtZXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHgAeA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}},"sample_logs":{"top_hits":{"from":0,"size":10,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["email"],"excludes":[]}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},pattern_count=COUNT($1),sample_logs=TAKE($0, $2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"patterns_field":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlHsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IGZhbHNlCn10AARleHBydAYZewogICJvcCI6IHsKICAgICJuYW1lIjogIkNBU0UiLAogICAgImtpbmQiOiAiQ0FTRSIsCiAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJTRUFSQ0giLAogICAgICAgICJraW5kIjogIlNFQVJDSCIsCiAgICAgICAgInN5bnRheCI6ICJJTlRFUk5BTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogewogICAgICAgICAgICAicmFuZ2VTZXQiOiBbCiAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgInNpbmdsZXRvbiIsCiAgICAgICAgICAgICAgICAie1widmFsdWVcIjpcIlwiLFwiY2hhcnNldE5hbWVcIjpcIklTTy04ODU5LTFcIixcImNvbGxhdGlvblwiOntcImNvbGxhdGlvbk5hbWVcIjpcIklTTy04ODU5LTEkZW5fVVMkcHJpbWFyeVwiLFwiY29lcmNpYmlsaXR5XCI6XCJJTVBMSUNJVFwiLFwibG9jYWxlXCI6XCJlbl9VU1wifSxcImNoYXJzZXRcIjpcIklTTy04ODU5LTFcIixcInZhbHVlQnl0ZXNcIjpudWxsfSIKICAgICAgICAgICAgICBdCiAgICAgICAgICAgIF0sCiAgICAgICAgICAgICJudWxsQXMiOiAiVFJVRSIKICAgICAgICAgIH0sCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogIiIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiUkVHRVhQX1JFUExBQ0UiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAiW2EtekEtWjAtOV0rIiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAiPCo+IiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQABWVtYWlsc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4ADH5xAH4AEHQABlNUUklOR35xAH4AFHQAB0tleXdvcmRxAH4AGXh4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"pattern_count":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlHsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IGZhbHNlCn10AARleHBydAYZewogICJvcCI6IHsKICAgICJuYW1lIjogIkNBU0UiLAogICAgImtpbmQiOiAiQ0FTRSIsCiAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJTRUFSQ0giLAogICAgICAgICJraW5kIjogIlNFQVJDSCIsCiAgICAgICAgInN5bnRheCI6ICJJTlRFUk5BTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogewogICAgICAgICAgICAicmFuZ2VTZXQiOiBbCiAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgInNpbmdsZXRvbiIsCiAgICAgICAgICAgICAgICAie1widmFsdWVcIjpcIlwiLFwiY2hhcnNldE5hbWVcIjpcIklTTy04ODU5LTFcIixcImNvbGxhdGlvblwiOntcImNvbGxhdGlvbk5hbWVcIjpcIklTTy04ODU5LTEkZW5fVVMkcHJpbWFyeVwiLFwiY29lcmNpYmlsaXR5XCI6XCJJTVBMSUNJVFwiLFwibG9jYWxlXCI6XCJlbl9VU1wifSxcImNoYXJzZXRcIjpcIklTTy04ODU5LTFcIixcInZhbHVlQnl0ZXNcIjpudWxsfSIKICAgICAgICAgICAgICBdCiAgICAgICAgICAgIF0sCiAgICAgICAgICAgICJudWxsQXMiOiAiVFJVRSIKICAgICAgICAgIH0sCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogIiIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiUkVHRVhQX1JFUExBQ0UiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAiW2EtekEtWjAtOV0rIiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAiPCo+IiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQABWVtYWlsc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4ADH5xAH4AEHQABlNUUklOR35xAH4AFHQAB0tleXdvcmRxAH4AGXh4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}},"sample_logs":{"top_hits":{"from":0,"size":10,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"email.keyword"}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_rare_usenull_false.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_rare_usenull_false.yaml new file mode 100644 index 00000000000..9c5157c72bc --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_rare_usenull_false.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(gender=[$0], state=[$1], count=[$2]) + LogicalFilter(condition=[<=($3, 2)]) + LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2)]) + LogicalAggregate(group=[{0, 1}], count=[COUNT()]) + LogicalProject(gender=[$4], state=[$7]) + LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count=COUNT()), RARE_TOP->rare 2 state by gender, PROJECT->[gender, state, count], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"gender":{"terms":{"field":"gender.keyword","size":10000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]},"aggregations":{"state":{"terms":{"field":"state.keyword","size":2,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"asc"},{"_key":"asc"}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_rare_usenull_true.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_rare_usenull_true.yaml new file mode 100644 index 00000000000..7bc4aa1e21c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_rare_usenull_true.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(gender=[$0], state=[$1], count=[$2]) + LogicalFilter(condition=[<=($3, 2)]) + LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2)]) + LogicalAggregate(group=[{0, 1}], count=[COUNT()]) + LogicalProject(gender=[$4], state=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[2], expr#5=[<=($t3, $t4)], proj#0..2=[{exprs}], $condition=[$t5]) + EnumerableWindow(window#0=[window(partition {0} order by [2] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_regex.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_regex.yaml index a8585cc3a25..a1c7349ac3f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_regex.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_regex.yaml @@ -6,4 +6,4 @@ calcite: LogicalFilter(condition=[REGEXP_CONTAINS($10, '^[A-Z][a-z]+$':VARCHAR)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->REGEXP_CONTAINS($10, '^[A-Z][a-z]+$':VARCHAR), LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQEj3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AUR7CiAgIm9wIjogewogICAgIm5hbWUiOiAiUkVHRVhQX0NPTlRBSU5TIiwKICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDEwLAogICAgICAibmFtZSI6ICIkMTAiCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICJeW0EtWl1bYS16XSskIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAt0AA5hY2NvdW50X251bWJlcn5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABExPTkd0AAlmaXJzdG5hbWVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AEHhwfnEAfgAKdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AAt0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAbeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAdAAAAAHNxAH4AAAAAAAN3BAAAAAJ0AAdrZXl3b3Jkc3EAfgARfnEAfgAKdAAGU1RSSU5HfnEAfgAXdAAHS2V5d29yZHEAfgAceHQAB2FkZHJlc3NzcQB+AA9xAH4AFXEAfgAYcQB+ABxzcQB+AAAAAAADdwQAAAAAeHQAB2JhbGFuY2VxAH4ADHQABmdlbmRlcnNxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAJxAH4AIXEAfgAieHQABGNpdHlzcQB+AA9xAH4AFXEAfgAYcQB+ABxzcQB+AAAAAAADdwQAAAACcQB+ACFxAH4AInh0AAhlbXBsb3llcnNxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAJxAH4AIXEAfgAieHQABXN0YXRlc3EAfgAPcQB+ABVxAH4AGHEAfgAcc3EAfgAAAAAAA3cEAAAAAnEAfgAhcQB+ACJ4dAADYWdlcQB+AAx0AAVlbWFpbHNxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAJxAH4AIXEAfgAieHQACGxhc3RuYW1lc3EAfgAPcQB+ABVxAH4AGHEAfgAcc3EAfgAAAAAAA3cEAAAAAnEAfgAhcQB+ACJ4eHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->REGEXP_CONTAINS($10, '^[A-Z][a-z]+$':VARCHAR), LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAl3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJsYXN0bmFtZSIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IGZhbHNlCn10AARleHBydAFCewogICJvcCI6IHsKICAgICJuYW1lIjogIlJFR0VYUF9DT05UQUlOUyIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiAwLAogICAgICAibmFtZSI6ICIkMCIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogIl5bQS1aXVthLXpdKyQiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQACGxhc3RuYW1lc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4ADH5xAH4AEHQABlNUUklOR35xAH4AFHQAB0tleXdvcmRxAH4AGXh4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_regex_negated.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_regex_negated.yaml index 61af415a3ca..e94f8ab11c3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_regex_negated.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_regex_negated.yaml @@ -6,4 +6,4 @@ calcite: LogicalFilter(condition=[NOT(REGEXP_CONTAINS($10, '.*son$':VARCHAR))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->NOT(REGEXP_CONTAINS($10, '.*son$':VARCHAR)), LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQEj3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AfV7CiAgIm9wIjogewogICAgIm5hbWUiOiAiTk9UIiwKICAgICJraW5kIjogIk5PVCIsCiAgICAic3ludGF4IjogIlBSRUZJWCIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIlJFR0VYUF9DT05UQUlOUyIsCiAgICAgICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiaW5wdXQiOiAxMCwKICAgICAgICAgICJuYW1lIjogIiQxMCIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogIi4qc29uJCIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAt0AA5hY2NvdW50X251bWJlcn5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABExPTkd0AAlmaXJzdG5hbWVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AEHhwfnEAfgAKdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AAt0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAbeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAdAAAAAHNxAH4AAAAAAAN3BAAAAAJ0AAdrZXl3b3Jkc3EAfgARfnEAfgAKdAAGU1RSSU5HfnEAfgAXdAAHS2V5d29yZHEAfgAceHQAB2FkZHJlc3NzcQB+AA9xAH4AFXEAfgAYcQB+ABxzcQB+AAAAAAADdwQAAAAAeHQAB2JhbGFuY2VxAH4ADHQABmdlbmRlcnNxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAJxAH4AIXEAfgAieHQABGNpdHlzcQB+AA9xAH4AFXEAfgAYcQB+ABxzcQB+AAAAAAADdwQAAAACcQB+ACFxAH4AInh0AAhlbXBsb3llcnNxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAJxAH4AIXEAfgAieHQABXN0YXRlc3EAfgAPcQB+ABVxAH4AGHEAfgAcc3EAfgAAAAAAA3cEAAAAAnEAfgAhcQB+ACJ4dAADYWdlcQB+AAx0AAVlbWFpbHNxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAJxAH4AIXEAfgAieHQACGxhc3RuYW1lc3EAfgAPcQB+ABVxAH4AGHEAfgAcc3EAfgAAAAAAA3cEAAAAAnEAfgAhcQB+ACJ4eHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->NOT(REGEXP_CONTAINS($10, '.*son$':VARCHAR)), LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAl3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJsYXN0bmFtZSIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IGZhbHNlCn10AARleHBydAHzewogICJvcCI6IHsKICAgICJuYW1lIjogIk5PVCIsCiAgICAia2luZCI6ICJOT1QiLAogICAgInN5bnRheCI6ICJQUkVGSVgiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJSRUdFWFBfQ09OVEFJTlMiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAiLipzb24kIiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQACGxhc3RuYW1lc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4ADH5xAH4AEHQABlNUUklOR35xAH4AFHQAB0tleXdvcmRxAH4AGXh4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_replace_command.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_replace_command.yaml new file mode 100644 index 00000000000..a867c569168 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_replace_command.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(state=[REPLACE($7, 'IL':VARCHAR, 'Illinois':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0=[{inputs}], expr#1=['IL':VARCHAR], expr#2=['Illinois':VARCHAR], expr#3=[REPLACE($t0, $t1, $t2)], $f0=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["state"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_replace_wildcard.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_replace_wildcard.yaml new file mode 100644 index 00000000000..0407849a472 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_replace_wildcard.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(state=[REGEXP_REPLACE($7, '^\Q\E(.*?)\QL\E$':VARCHAR, 'STATE_IL':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0=[{inputs}], expr#1=['^\Q\E(.*?)\QL\E$':VARCHAR], expr#2=['STATE_IL':VARCHAR], expr#3=[REGEXP_REPLACE($t0, $t1, $t2)], $f0=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["state"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_rex.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_rex.yaml index 3658fe35f41..d420fca0baf 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_rex.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_rex.yaml @@ -3,8 +3,8 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], initial=[$17]) LogicalSort(fetch=[5]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], initial=[REX_EXTRACT($10, '(?^[A-Z])', 1)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], initial=[REX_EXTRACT($10, '(?^[A-Z])', 'initial')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..10=[{inputs}], expr#11=['(?^[A-Z])'], expr#12=[1], expr#13=[REX_EXTRACT($t10, $t11, $t12)], proj#0..10=[{exprs}], $f11=[$t13]) + EnumerableCalc(expr#0..10=[{inputs}], expr#11=['(?^[A-Z])'], expr#12=['initial'], expr#13=[REX_EXTRACT($t10, $t11, $t12)], proj#0..10=[{exprs}], $f11=[$t13]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_time_range.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_time_range.json deleted file mode 100644 index 0399da3db52..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_time_range.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12])\n LogicalFilter(condition=[AND(>=($3, TIMESTAMP('2016-12-08 00:00:00.000000000':VARCHAR)), <($3, TIMESTAMP('2018-11-09 00:00:00.000000000':VARCHAR)))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], FILTER->SEARCH($3, Sarg[['2016-12-08 00:00:00':VARCHAR..'2018-11-09 00:00:00':VARCHAR)]:VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"birthdate\":{\"from\":\"2016-12-08T00:00:00.000Z\",\"to\":\"2018-11-09T00:00:00.000Z\",\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}},\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"birthdate\",\"gender\",\"city\",\"lastname\",\"balance\",\"employer\",\"state\",\"age\",\"email\",\"male\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_time_range.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_time_range.yaml new file mode 100644 index 00000000000..cfb07502429 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_time_range.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + LogicalFilter(condition=[AND(>=($3, TIMESTAMP('2016-12-08 00:00:00.000000000':VARCHAR)), <($3, TIMESTAMP('2018-11-09 00:00:00.000000000':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], FILTER->SEARCH($3, Sarg[['2016-12-08 00:00:00':VARCHAR..'2018-11-09 00:00:00':VARCHAR)]:VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"range":{"birthdate":{"from":"2016-12-08T00:00:00.000Z","to":"2018-11-09T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_correlated_subquery_in_select.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_correlated_subquery_in_select.yaml new file mode 100644 index 00000000000..39410dca951 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_correlated_subquery_in_select.yaml @@ -0,0 +1,20 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(variablesSet=[[$cor0]], id=[$2], name=[$0], count_dept=[$SCALAR_QUERY({ + LogicalAggregate(group=[{}], count(name)=[COUNT($0)]) + LogicalProject(name=[$0]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalFilter(condition=[=($cor0.id, $1)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) + })]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + physical: | + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[0:BIGINT], expr#6=[CASE($t4, $t5, $t3)], id=[$t1], name=[$t0], count_dept=[$t6]) + EnumerableLimit(fetch=[10000]) + EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($1, $2)], joinType=[left]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t2)], expr#4=[0], expr#5=[CASE($t3, $t2, $t4)], uid=[$t0], count(name)=[$t5]) + EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0})], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"id":{"terms":{"field":"id","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[PROJECT->[name, uid], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(name)=COUNT($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"uid","boost":1.0}},{"exists":{"field":"name","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["name","uid"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"uid":{"terms":{"field":"uid","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(name)":{"value_count":{"field":"name"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_correlated_subquery_in_where.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_correlated_subquery_in_where.yaml new file mode 100644 index 00000000000..3f4cb15d194 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_correlated_subquery_in_where.yaml @@ -0,0 +1,18 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(id=[$2], name=[$0]) + LogicalFilter(condition=[=($2, $SCALAR_QUERY({ + LogicalAggregate(group=[{}], max(uid)=[MAX($0)]) + LogicalProject(uid=[$1]) + LogicalFilter(condition=[=($cor0.id, $1)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) + }))], variablesSet=[[$cor0]]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], id=[$t1], name=[$t0]) + EnumerableHashJoin(condition=[=($1, $2)], joinType=[semi]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[=($t0, $t1)], proj#0..1=[{exprs}], $condition=[$t2]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[PROJECT->[uid], FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},max(uid)=MAX($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"uid","boost":1.0}},"_source":{"includes":["uid"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"uid1":{"terms":{"field":"uid","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(uid)":{"max":{"field":"uid"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_select.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_select.yaml new file mode 100644 index 00000000000..70fcf1c804d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_select.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(variablesSet=[[$cor0]], name=[$0], count_dept=[$SCALAR_QUERY({ + LogicalAggregate(group=[{}], count(name)=[COUNT($0)]) + LogicalProject(name=[$0]) + LogicalFilter(condition=[IS NOT NULL($0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) + })]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableNestedLoopJoin(condition=[true], joinType=[left]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count(name)=COUNT($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"name","boost":1.0}},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_where.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_where.yaml new file mode 100644 index 00000000000..a14c422cfb5 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_where.yaml @@ -0,0 +1,17 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(name=[$0]) + LogicalFilter(condition=[>($2, +($SCALAR_QUERY({ + LogicalAggregate(group=[{}], count(name)=[COUNT($0)]) + LogicalProject(name=[$0]) + LogicalFilter(condition=[IS NOT NULL($0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) + }), 999))], variablesSet=[[$cor0]]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], name=[$t0]) + EnumerableNestedLoopJoin(condition=[>($1, +($2, 999))], joinType=[inner]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count(name)=COUNT($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"name","boost":1.0}},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_script_push_on_text.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_script_push_on_text.json deleted file mode 100644 index bef1d8fc402..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_script_push_on_text.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], address_length=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(address_length=[CHAR_LENGTH($2)])\n LogicalFilter(condition=[>(CHAR_LENGTH($2), 0)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[address], SCRIPT->>(CHAR_LENGTH($0), 0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), address_length], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJhZGRyZXNzIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbN7CiAgIm9wIjogewogICAgIm5hbWUiOiAiPiIsCiAgICAia2luZCI6ICJHUkVBVEVSX1RIQU4iLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAgICAgImtpbmQiOiAiQ0hBUl9MRU5HVEgiLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgIm5hbWUiOiAiJDAiCiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AAdhZGRyZXNzc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAAAeHh4\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},\"_source\":{\"includes\":[\"address\"],\"excludes\":[]},\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"address_length\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJhZGRyZXNzIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AKZ7CiAgIm9wIjogewogICAgIm5hbWUiOiAiQ0hBUl9MRU5HVEgiLAogICAgImtpbmQiOiAiQ0hBUl9MRU5HVEgiLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAReHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABx4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB4AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABJ+cQB+AAt0AAZTVFJJTkd+cQB+ABh0AAdLZXl3b3JkcQB+AB14dAAHYWRkcmVzc3NxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgANdAAGZ2VuZGVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAEY2l0eXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGVtcGxveWVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAFc3RhdGVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AANhZ2VxAH4ADXQABWVtYWlsc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIbGFzdG5hbWVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h4AHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_script_push_on_text.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_script_push_on_text.yaml new file mode 100644 index 00000000000..87f2c9ab813 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_script_push_on_text.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], address_length=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(address_length=[CHAR_LENGTH($2)]) + LogicalFilter(condition=[>(CHAR_LENGTH($2), 0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[address], SCRIPT->>(CHAR_LENGTH($0), 0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), address_length], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJhZGRyZXNzIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbN7CiAgIm9wIjogewogICAgIm5hbWUiOiAiPiIsCiAgICAia2luZCI6ICJHUkVBVEVSX1RIQU4iLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAgICAgImtpbmQiOiAiQ0hBUl9MRU5HVEgiLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgIm5hbWUiOiAiJDAiCiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AAdhZGRyZXNzc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAAAeHh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},"_source":{"includes":["address"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"address_length":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJhZGRyZXNzIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AKZ7CiAgIm9wIjogewogICAgIm5hbWUiOiAiQ0hBUl9MRU5HVEgiLAogICAgImtpbmQiOiAiQ0hBUl9MRU5HVEgiLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAAHYWRkcmVzc3NyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgALeHB+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4AEXQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABh4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+ABoAAAAAc3EAfgAAAAAAA3cEAAAAAHh4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":true,"value_type":"long","missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.json index 9a2551446db..5844a594d72 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.json @@ -1,6 +1,7 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(firstname=[$1], age=[$8], address=[$2])\n LogicalFilter(condition=[AND(=($2, '671 Bristol Street'), =(-($8, 2), 30))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, address, age], SCRIPT->AND(=($1, '671 Bristol Street'), =(-($2, 2), 30)), PROJECT->[firstname, age, address], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"must\":[{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"EQUALS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"input\\\\\\\": 1,\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"$1\\\\\\\"\\\\n },\\\\n {\\\\n \\\\\\\"literal\\\\\\\": \\\\\\\"671 Bristol Street\\\\\\\",\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"VARCHAR\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": false,\\\\n \\\\\\\"precision\\\\\\\": -1\\\\n }\\\\n }\\\\n ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"EQUALS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"-\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"MINUS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"input\\\\\\\": 2,\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"$2\\\\\\\"\\\\n },\\\\n {\\\\n \\\\\\\"literal\\\\\\\": 2,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"INTEGER\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": false\\\\n }\\\\n }\\\\n ],\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"literal\\\\\\\": 30,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"INTEGER\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": false\\\\n }\\\\n }\\\\n ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"firstname\",\"age\",\"address\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, address, age], SCRIPT->AND(=($1, '671 Bristol Street'), =(-($2, 2), 30)), PROJECT->[firstname, age, address], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"must\":[{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"EQUALS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"input\\\\\\\": 0,\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"$0\\\\\\\"\\\\n },\\\\n {\\\\n \\\\\\\"literal\\\\\\\": \\\\\\\"671 Bristol Street\\\\\\\",\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"VARCHAR\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": false,\\\\n \\\\\\\"precision\\\\\\\": -1\\\\n }\\\\n }\\\\n ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"EQUALS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"-\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"MINUS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"input\\\\\\\": 0,\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"$0\\\\\\\"\\\\n },\\\\n {\\\\n \\\\\\\"literal\\\\\\\": 2,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"INTEGER\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": false\\\\n }\\\\n }\\\\n ],\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"literal\\\\\\\": 30,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"INTEGER\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": false\\\\n }\\\\n }\\\\n ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"firstname\",\"age\",\"address\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n", + "extended": "public org.apache.calcite.linq4j.Enumerable bind(final org.apache.calcite.DataContext root) {\n final org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan v1stashed = (org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan) root.get(\"v1stashed\");\n return v1stashed.scan();\n}\n\n\npublic Class getElementType() {\n return java.lang.Object[].class;\n}\n\n\n" } -} +} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time.yaml index cb3428897ac..5c7850cdf5e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time.yaml @@ -7,5 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, events]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[0], expr#3=[>($t1, $t2)], count()=[$t1], @timestamp=[$t0], $condition=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null},"aggregations":{"count()":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), @timestamp]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time2.yaml index a0080e88f90..4efe38c96d1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time2.yaml @@ -7,5 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, events]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NOT NULL($t1)], avg(cpu_usage)=[$t1], @timestamp=[$t0], $condition=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(cpu_usage)=AVG($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null},"aggregations":{"avg(cpu_usage)":{"avg":{"field":"cpu_usage"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(cpu_usage)=AVG($1)), PROJECT->[avg(cpu_usage), @timestamp]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null},"aggregations":{"avg(cpu_usage)":{"avg":{"field":"cpu_usage"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term.yaml new file mode 100644 index 00000000000..14cf8e6db82 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$2], @timestamp=[$0], region=[$1]) + LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) + LogicalProject(@timestamp=[$15], region=[$7]) + LogicalFilter(condition=[AND(IS NOT NULL($15), IS NOT NULL($7))]) + LogicalProject(environment=[$0], status_code=[$2], service=[$3], host=[$4], memory_usage=[$5], response_time=[$6], cpu_usage=[$7], region=[$8], bytes_sent=[$9], _id=[$10], _index=[$11], _score=[$12], _maxscore=[$13], _sort=[$14], _routing=[$15], @timestamp=[WIDTH_BUCKET($1, 3, -(MAX($1) OVER (), MIN($1) OVER ()), MAX($1) OVER ())]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), @timestamp, region], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"region":{"terms":{"field":"region","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term2.yaml new file mode 100644 index 00000000000..1dc48f5a550 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term2.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg(cpu_usage)=[$2], @timestamp=[$0], region=[$1]) + LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[AVG($2)]) + LogicalProject(@timestamp=[$15], region=[$7], cpu_usage=[$6]) + LogicalFilter(condition=[AND(IS NOT NULL($15), IS NOT NULL($7))]) + LogicalProject(environment=[$0], status_code=[$2], service=[$3], host=[$4], memory_usage=[$5], response_time=[$6], cpu_usage=[$7], region=[$8], bytes_sent=[$9], _id=[$10], _index=[$11], _score=[$12], _maxscore=[$13], _sort=[$14], _routing=[$15], @timestamp=[WIDTH_BUCKET($1, 3, -(MAX($1) OVER (), MIN($1) OVER ()), MAX($1) OVER ())]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1, 2},avg(cpu_usage)=AVG($0)), PROJECT->[avg(cpu_usage), @timestamp, region], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"region":{"terms":{"field":"region","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null},"aggregations":{"avg(cpu_usage)":{"avg":{"field":"cpu_usage"}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_dc.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_dc.yaml new file mode 100644 index 00000000000..9dd91501bf8 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_dc.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], distinct_states=[DISTINCT_COUNT_APPROX($7) OVER (ROWS UNBOUNDED PRECEDING)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [DISTINCT_COUNT_APPROX($7)])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_distinct_count.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_distinct_count.yaml new file mode 100644 index 00000000000..32538ab17df --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_distinct_count.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], distinct_states=[$18]) + LogicalSort(sort0=[$17], dir0=[ASC]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], distinct_states=[DISTINCT_COUNT_APPROX($7) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..12=[{inputs}], proj#0..10=[{exprs}], distinct_states=[$t12]) + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$11], dir0=[ASC]) + EnumerableWindow(window#0=[window(partition {4} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [DISTINCT_COUNT_APPROX($7)])]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_earliest_latest.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_earliest_latest.yaml new file mode 100644 index 00000000000..cac21b929ee --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_earliest_latest.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[$12], latest_message=[$13]) + LogicalSort(sort0=[$11], dir0=[ASC]) + LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], _id=[$5], _index=[$6], _score=[$7], _maxscore=[$8], _sort=[$9], _routing=[$10], __stream_seq__=[$11], earliest_message=[ARG_MIN($3, $2) OVER (PARTITION BY $1 ROWS UNBOUNDED PRECEDING)], latest_message=[ARG_MAX($3, $2) OVER (PARTITION BY $1 ROWS UNBOUNDED PRECEDING)]) + LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], _id=[$5], _index=[$6], _score=[$7], _maxscore=[$8], _sort=[$9], _routing=[$10], __stream_seq__=[ROW_NUMBER() OVER ()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) + physical: | + EnumerableCalc(expr#0..7=[{inputs}], proj#0..4=[{exprs}], earliest_message=[$t6], latest_message=[$t7]) + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$5], dir0=[ASC]) + EnumerableWindow(window#0=[window(partition {1} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ARG_MIN($3, $2), ARG_MAX($3, $2)])]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[PROJECT->[created_at, server, @timestamp, message, level]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["created_at","server","@timestamp","message","level"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_earliest_latest_custom_time.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_earliest_latest_custom_time.yaml new file mode 100644 index 00000000000..f19625d85e5 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_earliest_latest_custom_time.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[$12], latest_message=[$13]) + LogicalSort(sort0=[$11], dir0=[ASC]) + LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], _id=[$5], _index=[$6], _score=[$7], _maxscore=[$8], _sort=[$9], _routing=[$10], __stream_seq__=[$11], earliest_message=[ARG_MIN($3, $0) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING)], latest_message=[ARG_MAX($3, $0) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING)]) + LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], _id=[$5], _index=[$6], _score=[$7], _maxscore=[$8], _sort=[$9], _routing=[$10], __stream_seq__=[ROW_NUMBER() OVER ()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) + physical: | + EnumerableCalc(expr#0..7=[{inputs}], proj#0..4=[{exprs}], earliest_message=[$t6], latest_message=[$t7]) + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$5], dir0=[ASC]) + EnumerableWindow(window#0=[window(partition {4} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ARG_MIN($3, $0), ARG_MAX($3, $0)])]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[PROJECT->[created_at, server, @timestamp, message, level]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["created_at","server","@timestamp","message","level"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_earliest_latest_no_group.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_earliest_latest_no_group.yaml new file mode 100644 index 00000000000..f17643ab804 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_earliest_latest_no_group.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[ARG_MIN($3, $2) OVER (ROWS UNBOUNDED PRECEDING)], latest_message=[ARG_MAX($3, $2) OVER (ROWS UNBOUNDED PRECEDING)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ARG_MIN($3, $2), ARG_MAX($3, $2)])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[PROJECT->[created_at, server, @timestamp, message, level]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["created_at","server","@timestamp","message","level"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml new file mode 100644 index 00000000000..293dd785f96 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml @@ -0,0 +1,29 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$18]) + LogicalSort(sort0=[$17], dir0=[ASC]) + LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{4, 17}]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalAggregate(group=[{}], avg_age=[AVG($8)]) + LogicalFilter(condition=[AND(>=($17, -($cor0.__stream_seq__, 1)), <=($17, $cor0.__stream_seq__), =($4, $cor0.gender))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t16]) + EnumerableLimit(fetch=[10000]) + EnumerableHashJoin(condition=[AND(=($4, $13), =($11, $14), =($12, $15))], joinType=[left]) + EnumerableSort(sort0=[$11], dir0=[ASC]) + EnumerableCalc(expr#0..11=[{inputs}], expr#12=[1], expr#13=[-($t11, $t12)], proj#0..11=[{exprs}], $f12=[$t13]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t4, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t3)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t4)], proj#0..2=[{exprs}], avg_age=[$t10]) + EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($4)], agg#1=[COUNT($4)]) + EnumerableHashJoin(condition=[AND(=($0, $3), >=($5, $2), <=($5, $1))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2}]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[-($t1, $t2)], proj#0..1=[{exprs}], $f2=[$t3]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml new file mode 100644 index 00000000000..0e8ed3a3dde --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml @@ -0,0 +1,38 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$21]) + LogicalSort(sort0=[$17], dir0=[ASC]) + LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{4, 17, 20}]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), COALESCE(SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalAggregate(group=[{}], avg_age=[AVG($8)]) + LogicalFilter(condition=[AND(<($17, $cor0.__stream_seq__), =($20, $cor0.__seg_id__), =($4, $cor0.gender))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), COALESCE(SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t16]) + EnumerableLimit(fetch=[10000]) + EnumerableHashJoin(condition=[AND(=($4, $13), =($11, $14), =($12, $15))], joinType=[left]) + EnumerableSort(sort0=[$11], dir0=[ASC]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[0], expr#17=[COALESCE($t15, $t16)], expr#18=[+($t14, $t17)], proj#0..11=[{exprs}], __seg_id__=[$t18]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($12)])], window#1=[window(rows between UNBOUNDED PRECEDING and $14 PRECEDING aggs [$SUM0($13)])], constants=[[1]]) + EnumerableCalc(expr#0..11=[{inputs}], expr#12=[34], expr#13=[>($t8, $t12)], expr#14=[1], expr#15=[0], expr#16=[CASE($t13, $t14, $t15)], expr#17=[25], expr#18=[<($t8, $t17)], expr#19=[CASE($t18, $t14, $t15)], proj#0..11=[{exprs}], __reset_before_flag__=[$t16], __reset_after_flag__=[$t19]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t4, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t3)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t4)], proj#0..2=[{exprs}], avg_age=[$t10]) + EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($4)], agg#1=[COUNT($4)]) + EnumerableHashJoin(condition=[AND(=($2, $6), =($0, $3), <($5, $1))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2}]) + EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[COALESCE($t5, $t6)], expr#8=[+($t4, $t7)], proj#0..1=[{exprs}], __seg_id__=[$t8]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $4 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_take.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_take.json deleted file mode 100644 index a48633f7ea2..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_take.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], take=[TAKE($0, $1)])\n LogicalProject(firstname=[$1], $f1=[2])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},take=TAKE($0, $1))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"take\":{\"top_hits\":{\"from\":0,\"size\":2,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_take.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_take.yaml new file mode 100644 index 00000000000..e395f0e7485 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_take.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], take=[TAKE($0, $1)]) + LogicalProject(firstname=[$1], $f1=[2]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},take=TAKE($0, $1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"take":{"top_hits":{"from":0,"size":2,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"firstname.keyword"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function.json deleted file mode 100644 index 6c6db662cbc..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10])\n LogicalFilter(condition=[ILIKE($2, '%Holmes%':VARCHAR, '\\')])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->ILIKE($2, '%Holmes%':VARCHAR, '\\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQEj3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0Aa57CiAgIm9wIjogewogICAgIm5hbWUiOiAiSUxJS0UiLAogICAgImtpbmQiOiAiTElLRSIsCiAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDIsCiAgICAgICJuYW1lIjogIiQyIgogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAiJUhvbG1lcyUiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAiXFwiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAALdAAOYWNjb3VudF9udW1iZXJ+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05HdAAJZmlyc3RuYW1lc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+ABB4cH5xAH4ACnQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgALdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AG3hwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AHQAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AEX5xAH4ACnQABlNUUklOR35xAH4AF3QAB0tleXdvcmRxAH4AHHh0AAdhZGRyZXNzc3EAfgAPcQB+ABVxAH4AGHEAfgAcc3EAfgAAAAAAA3cEAAAAAHh0AAdiYWxhbmNlcQB+AAx0AAZnZW5kZXJzcQB+AA9xAH4AFXEAfgAYcQB+ABxzcQB+AAAAAAADdwQAAAACcQB+ACFxAH4AInh0AARjaXR5c3EAfgAPcQB+ABVxAH4AGHEAfgAcc3EAfgAAAAAAA3cEAAAAAnEAfgAhcQB+ACJ4dAAIZW1wbG95ZXJzcQB+AA9xAH4AFXEAfgAYcQB+ABxzcQB+AAAAAAADdwQAAAACcQB+ACFxAH4AInh0AAVzdGF0ZXNxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAJxAH4AIXEAfgAieHQAA2FnZXEAfgAMdAAFZW1haWxzcQB+AA9xAH4AFXEAfgAYcQB+ABxzcQB+AAAAAAADdwQAAAACcQB+ACFxAH4AInh0AAhsYXN0bmFtZXNxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAJxAH4AIXEAfgAieHh4\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function.yaml new file mode 100644 index 00000000000..c3f7ffe2a93 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalFilter(condition=[ILIKE($2, '%Holmes%', '\')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->ILIKE($2, '%Holmes%', '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJhZGRyZXNzIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0Aap7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSUxJS0UiLAogICAgImtpbmQiOiAiTElLRSIsCiAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDAsCiAgICAgICJuYW1lIjogIiQwIgogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAiJUhvbG1lcyUiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogOAogICAgICB9CiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICJcXCIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAxCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AAdhZGRyZXNzc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAAAeHh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml index 23206474020..adba9c12202 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml @@ -46,30 +46,26 @@ calcite: EnumerableUnion(all=[false]) EnumerableAggregate(group=[{0, 1}], actual_count=[$SUM0($2)]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t0):TIMESTAMP(0) NOT NULL], expr#6=[IS NOT NULL($t3)], expr#7=[IS NULL($t1)], expr#8=[null:NULL], expr#9=['OTHER'], expr#10=[CASE($t7, $t8, $t9)], expr#11=[CASE($t6, $t1, $t10)], @timestamp=[$t5], host=[$t11], count=[$t2]) - EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], @timestamp=[$t1], host=[$t0], $f2_0=[$t2]) - EnumerableAggregate(group=[{0, 1}], agg#0=[COUNT()]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=['m'], expr#4=[SPAN($t1, $t2, $t3)], host=[$t0], $f1=[$t4]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["host","@timestamp"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableLimit(fetch=[10]) - EnumerableSort(sort0=[$1], dir0=[DESC]) - EnumerableAggregate(group=[{0}], grand_total=[COUNT()]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, @timestamp], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"host","boost":1.0}},"_source":{"includes":["host","@timestamp"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[left]) + EnumerableCalc(expr#0..2=[{inputs}], @timestamp=[$t1], host=[$t0], $f2_0=[$t2]) + EnumerableAggregate(group=[{0, 1}], agg#0=[COUNT()]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=['m'], expr#4=[SPAN($t1, $t2, $t3)], host=[$t0], $f1=[$t4]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["host","@timestamp"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableLimit(fetch=[10]) + EnumerableSort(sort0=[$1], dir0=[DESC]) + EnumerableAggregate(group=[{0}], grand_total=[COUNT()]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, @timestamp], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"host","boost":1.0}},"_source":{"includes":["host","@timestamp"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):TIMESTAMP(0) NOT NULL], expr#3=[0], @timestamp=[$t2], host=[$t1], count=[$t3]) EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1})], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"$f2":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableAggregate(group=[{0}]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t1)], expr#4=[IS NULL($t0)], expr#5=[null:NULL], expr#6=['OTHER'], expr#7=[CASE($t4, $t5, $t6)], expr#8=[CASE($t3, $t0, $t7)], $f0=[$t8]) - EnumerableMergeJoin(condition=[=($0, $1)], joinType=[left]) - EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..1=[{inputs}], host=[$t0]) - EnumerableAggregate(group=[{0, 1}]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=['m'], expr#4=[SPAN($t1, $t2, $t3)], host=[$t0], $f1=[$t4]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["host","@timestamp"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableLimit(fetch=[10]) - EnumerableSort(sort0=[$1], dir0=[DESC]) - EnumerableAggregate(group=[{0}], grand_total=[COUNT()]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, @timestamp], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"host","boost":1.0}},"_source":{"includes":["host","@timestamp"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + EnumerableCalc(expr#0..1=[{inputs}], host=[$t0]) + EnumerableAggregate(group=[{0, 1}]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=['m'], expr#4=[SPAN($t1, $t2, $t3)], host=[$t0], $f1=[$t4]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["host","@timestamp"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableLimit(fetch=[10]) + EnumerableSort(sort0=[$1], dir0=[DESC]) + EnumerableAggregate(group=[{0}], grand_total=[COUNT()]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, @timestamp], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"host","boost":1.0}},"_source":{"includes":["host","@timestamp"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_top_k_then_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_top_k_then_sort_push.yaml new file mode 100644 index 00000000000..ec71a9d1130 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_top_k_then_sort_push.yaml @@ -0,0 +1,17 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalSort(sort0=[$8], dir0=[ASC-nulls-first]) + LogicalSort(sort0=[$3], dir0=[ASC-nulls-first], fetch=[5]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], age=[$t1]) + EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[balance, age], SORT->[{ + "balance" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["balance","age"],"excludes":[]},"sort":[{"balance":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_top_usenull_false.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_top_usenull_false.yaml new file mode 100644 index 00000000000..21457c6170f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_top_usenull_false.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(gender=[$0], state=[$1], count=[$2]) + LogicalFilter(condition=[<=($3, 2)]) + LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 DESC)]) + LogicalAggregate(group=[{0, 1}], count=[COUNT()]) + LogicalProject(gender=[$4], state=[$7]) + LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count=COUNT()), RARE_TOP->top 2 state by gender, PROJECT->[gender, state, count], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"gender":{"terms":{"field":"gender.keyword","size":10000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]},"aggregations":{"state":{"terms":{"field":"state.keyword","size":2,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_top_usenull_true.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_top_usenull_true.yaml new file mode 100644 index 00000000000..51ffb883407 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_top_usenull_true.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(gender=[$0], state=[$1], count=[$2]) + LogicalFilter(condition=[<=($3, 2)]) + LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 DESC)]) + LogicalAggregate(group=[{0, 1}], count=[COUNT()]) + LogicalProject(gender=[$4], state=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[2], expr#5=[<=($t3, $t4)], proj#0..2=[{exprs}], $condition=[$t5]) + EnumerableWindow(window#0=[window(partition {0} order by [2 DESC] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_push.yaml index fc9c57db158..683bfe610cd 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_push.yaml @@ -8,6 +8,6 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], expr#5=[>($t1, $t4)], expr#6=[CAST($t3):DOUBLE NOT NULL], expr#7=[/($t2, $t6)], expr#8=[null:NULL], expr#9=[CASE($t5, $t7, $t8)], ageTrend=[$t9]) - EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), $SUM0($0), COUNT($0)])]) + EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), $SUM0($0), COUNT($0)])], constants=[[1]]) EnumerableCalc(expr#0=[{inputs}], expr#1=[IS NOT NULL($t0)], age=[$t0], $condition=[$t1]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_sort_push.yaml index 2003143e673..94265227c8e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_sort_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_sort_push.yaml @@ -9,7 +9,11 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], expr#5=[>($t1, $t4)], expr#6=[CAST($t3):DOUBLE NOT NULL], expr#7=[/($t2, $t6)], expr#8=[null:NULL], expr#9=[CASE($t5, $t7, $t8)], ageTrend=[$t9]) - EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), $SUM0($0), COUNT($0)])]) + EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), $SUM0($0), COUNT($0)])], constants=[[1]]) EnumerableCalc(expr#0=[{inputs}], expr#1=[IS NOT NULL($t0)], age=[$t0], $condition=[$t1]) - EnumerableSort(sort0=[$0], dir0=[ASC]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->5, SORT->[{ + "age" : { + "order" : "asc", + "missing" : "_last" + } + }]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]},"sort":[{"age":{"order":"asc","missing":"_last"}}]}, requestedTotalSize=5, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/keyword_in_range.yaml b/integ-test/src/test/resources/expectedOutput/calcite/keyword_in_range.yaml new file mode 100644 index 00000000000..85c08cf100c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/keyword_in_range.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)))]) + LogicalFilter(condition=[query_string(MAP('query', 'process.name:kernel':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->AND(query_string(MAP('query', 'process.name:kernel':VARCHAR)), SEARCH($7, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR)]:VARCHAR)), LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"must":[{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/keyword_terms.yaml b/integ-test/src/test/resources/expectedOutput/calcite/keyword_terms.yaml new file mode 100644 index 00000000000..da777dc2784 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/keyword_terms.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[500]) + LogicalProject(station=[$1], aws.cloudwatch.log_stream=[$0]) + LogicalAggregate(group=[{0}], station=[COUNT()]) + LogicalProject(aws.cloudwatch.log_stream=[$34]) + LogicalFilter(condition=[IS NOT NULL($34)]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},station=COUNT()), PROJECT->[station, aws.cloudwatch.log_stream], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->500, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","size":500,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"station":"desc"},{"_key":"asc"}]},"aggregations":{"station":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/keyword_terms_low_cardinality.yaml b/integ-test/src/test/resources/expectedOutput/calcite/keyword_terms_low_cardinality.yaml new file mode 100644 index 00000000000..fd4f1b547e3 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/keyword_terms_low_cardinality.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[50]) + LogicalProject(country=[$1], aws.cloudwatch.log_stream=[$0]) + LogicalAggregate(group=[{0}], country=[COUNT()]) + LogicalProject(aws.cloudwatch.log_stream=[$34]) + LogicalFilter(condition=[IS NOT NULL($34)]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},country=COUNT()), PROJECT->[country, aws.cloudwatch.log_stream], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->50, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","size":50,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"country":"desc"},{"_key":"asc"}]},"aggregations":{"country":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/multi_terms_keyword.yaml b/integ-test/src/test/resources/expectedOutput/calcite/multi_terms_keyword.yaml new file mode 100644 index 00000000000..8ce21fba101 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/multi_terms_keyword.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(count()=[$2], process.name=[$0], cloud.region=[$1]) + LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) + LogicalProject(process.name=[$7], cloud.region=[$14]) + LogicalFilter(condition=[AND(IS NOT NULL($7), IS NOT NULL($14))]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-05 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-05 05:00:00':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[process.name, cloud.region, @timestamp], FILTER->SEARCH($2, Sarg[['2023-01-05 00:00:00':VARCHAR..'2023-01-05 05:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[count(), process.name, cloud.region], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-05T00:00:00.000Z","to":"2023-01-05T05:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["process.name","cloud.region","@timestamp"],"excludes":[]},"aggregations":{"process.name|cloud.region":{"multi_terms":{"terms":[{"field":"process.name"},{"field":"cloud.region"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]},"aggregations":{"count()":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/query_string_on_message.yaml b/integ-test/src/test/resources/expectedOutput/calcite/query_string_on_message.yaml new file mode 100644 index 00000000000..31cbb3b8d70 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/query_string_on_message.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + LogicalFilter(condition=[query_string(MAP('query', '((message:monkey OR message:jackal) OR message:bear)':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->query_string(MAP('query', '((message:monkey OR message:jackal) OR message:bear)':VARCHAR)), LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"((message:monkey OR message:jackal) OR message:bear)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/query_string_on_message_filtered.yaml b/integ-test/src/test/resources/expectedOutput/calcite/query_string_on_message_filtered.yaml new file mode 100644 index 00000000000..e1471d87a4e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/query_string_on_message_filtered.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 10:00:00':VARCHAR)), query_string(MAP('fields', MAP('message':VARCHAR, 1.0E0:DOUBLE)), MAP('query', 'monkey jackal bear':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->AND(SEARCH($7, Sarg[['2023-01-03 00:00:00':VARCHAR..'2023-01-03 10:00:00':VARCHAR)]:VARCHAR), query_string(MAP('fields', MAP('message':VARCHAR, 1.0E0:DOUBLE)), MAP('query', 'monkey jackal bear':VARCHAR))), LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2023-01-03T00:00:00.000Z","to":"2023-01-03T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"query_string":{"query":"monkey jackal bear","fields":["message^1.0"],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/query_string_on_message_filtered_sorted_num.yaml b/integ-test/src/test/resources/expectedOutput/calcite/query_string_on_message_filtered_sorted_num.yaml new file mode 100644 index 00000000000..27a43886bc4 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/query_string_on_message_filtered_sorted_num.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 10:00:00':VARCHAR)), query_string(MAP('fields', MAP('message':VARCHAR, 1.0E0:DOUBLE)), MAP('query', 'monkey jackal bear':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->AND(SEARCH($7, Sarg[['2023-01-03 00:00:00':VARCHAR..'2023-01-03 10:00:00':VARCHAR)]:VARCHAR), query_string(MAP('fields', MAP('message':VARCHAR, 1.0E0:DOUBLE)), MAP('query', 'monkey jackal bear':VARCHAR))), SORT->[{ + "@timestamp" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2023-01-03T00:00:00.000Z","to":"2023-01-03T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"query_string":{"query":"monkey jackal bear","fields":["message^1.0"],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range.yaml new file mode 100644 index 00000000000..56c63c5c406 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->SEARCH($7, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR)]:VARCHAR), LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range_agg_1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range_agg_1.yaml new file mode 100644 index 00000000000..86c1551609c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range_agg_1.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], range_bucket=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(range_bucket=[CASE(<($28, -10), 'range_1':VARCHAR, SEARCH($28, Sarg[[-10..10)]), 'range_2':VARCHAR, SEARCH($28, Sarg[[10..100)]), 'range_3':VARCHAR, SEARCH($28, Sarg[[100..1000)]), 'range_4':VARCHAR, SEARCH($28, Sarg[[1000..2000)]), 'range_5':VARCHAR, >=($28, 2000), 'range_6':VARCHAR, null:NULL)]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), range_bucket]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"range_bucket":{"range":{"field":"metrics.size","ranges":[{"key":"range_1","to":-10.0},{"key":"range_2","from":-10.0,"to":10.0},{"key":"range_3","from":10.0,"to":100.0},{"key":"range_4","from":100.0,"to":1000.0},{"key":"range_5","from":1000.0,"to":2000.0},{"key":"range_6","from":2000.0}],"keyed":true}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range_agg_2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range_agg_2.yaml new file mode 100644 index 00000000000..daae2d2fc97 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range_agg_2.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], range_bucket=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(range_bucket=[CASE(<($28, 100), 'range_1':VARCHAR, SEARCH($28, Sarg[[100..1000)]), 'range_2':VARCHAR, SEARCH($28, Sarg[[1000..2000)]), 'range_3':VARCHAR, >=($28, 2000), 'range_4':VARCHAR, null:NULL)]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), range_bucket]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"range_bucket":{"range":{"field":"metrics.size","ranges":[{"key":"range_1","to":100.0},{"key":"range_2","from":100.0,"to":1000.0},{"key":"range_3","from":1000.0,"to":2000.0},{"key":"range_4","from":2000.0}],"keyed":true}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range_auto_date_histo.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range_auto_date_histo.yaml new file mode 100644 index 00000000000..21c20b5a523 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range_auto_date_histo.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$2], range_bucket=[$0], @timestamp=[$1]) + LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) + LogicalProject(range_bucket=[CASE(<($28, -10), 'range_1':VARCHAR, SEARCH($28, Sarg[[-10..10)]), 'range_2':VARCHAR, SEARCH($28, Sarg[[10..100)]), 'range_3':VARCHAR, SEARCH($28, Sarg[[100..1000)]), 'range_4':VARCHAR, SEARCH($28, Sarg[[1000..2000)]), 'range_5':VARCHAR, >=($28, 2000), 'range_6':VARCHAR, null:NULL)], @timestamp=[WIDTH_BUCKET($17, 20, -(MAX($17) OVER (), MIN($17) OVER ()), MAX($17) OVER ())]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), range_bucket, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"range_bucket":{"range":{"field":"metrics.size","ranges":[{"key":"range_1","to":-10.0},{"key":"range_2","from":-10.0,"to":10.0},{"key":"range_3","from":10.0,"to":100.0},{"key":"range_4","from":100.0,"to":1000.0},{"key":"range_5","from":1000.0,"to":2000.0},{"key":"range_6","from":2000.0}],"keyed":true},"aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":20,"minimum_interval":null}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range_auto_date_histo_with_metrics.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range_auto_date_histo_with_metrics.yaml new file mode 100644 index 00000000000..b29b215e612 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range_auto_date_histo_with_metrics.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(tmin=[$2], tavg=[$3], tmax=[$4], range_bucket=[$0], @timestamp=[$1]) + LogicalAggregate(group=[{0, 1}], tmin=[MIN($2)], tavg=[AVG($3)], tmax=[MAX($3)]) + LogicalProject(range_bucket=[CASE(<($28, 100), 'range_1':VARCHAR, SEARCH($28, Sarg[[100..1000)]), 'range_2':VARCHAR, SEARCH($28, Sarg[[1000..2000)]), 'range_3':VARCHAR, >=($28, 2000), 'range_4':VARCHAR, null:NULL)], @timestamp=[WIDTH_BUCKET($17, 10, -(MAX($17) OVER (), MIN($17) OVER ()), MAX($17) OVER ())], metrics.tmin=[$29], metrics.size=[$28]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},tmin=MIN($2),tavg=AVG($3),tmax=MAX($3)), PROJECT->[tmin, tavg, tmax, range_bucket, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"range_bucket":{"range":{"field":"metrics.size","ranges":[{"key":"range_1","to":100.0},{"key":"range_2","from":100.0,"to":1000.0},{"key":"range_3","from":1000.0,"to":2000.0},{"key":"range_4","from":2000.0}],"keyed":true},"aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":10,"minimum_interval":null},"aggregations":{"tmin":{"min":{"field":"metrics.tmin"}},"tavg":{"avg":{"field":"metrics.size"}},"tmax":{"max":{"field":"metrics.size"}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range_field_conjunction_big_range_big_term_query.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range_field_conjunction_big_range_big_term_query.yaml new file mode 100644 index 00000000000..ba8b035ab51 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range_field_conjunction_big_range_big_term_query.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + LogicalFilter(condition=[AND(=($7, 'systemd'), SEARCH($28, Sarg[[1..100]]))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, process.name, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], FILTER->AND(=($2, 'systemd'), SEARCH($14, Sarg[[1..100]])), LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"must":[{"term":{"process.name":{"value":"systemd","boost":1.0}}},{"range":{"metrics.size":{"from":1.0,"to":100.0,"include_lower":true,"include_upper":true,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range_field_conjunction_small_range_big_term_query.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range_field_conjunction_small_range_big_term_query.yaml new file mode 100644 index 00000000000..69dddd2ef14 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range_field_conjunction_small_range_big_term_query.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + LogicalFilter(condition=[SEARCH($28, Sarg[[20..30]])]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], FILTER->SEARCH($13, Sarg[[20..30]]), LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"range":{"metrics.size":{"from":20.0,"to":30.0,"include_lower":true,"include_upper":true,"boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range_field_conjunction_small_range_small_term_query.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range_field_conjunction_small_range_small_term_query.yaml new file mode 100644 index 00000000000..612e412b307 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range_field_conjunction_small_range_small_term_query.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + LogicalFilter(condition=[OR(=($34, 'indigodagger'), SEARCH($28, Sarg[[10..20]]))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, aws.cloudwatch.log_stream, event], FILTER->OR(=($15, 'indigodagger'), SEARCH($13, Sarg[[10..20]])), LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"should":[{"term":{"aws.cloudwatch.log_stream":{"value":"indigodagger","boost":1.0}}},{"range":{"metrics.size":{"from":10.0,"to":20.0,"include_lower":true,"include_upper":true,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range_field_disjunction_big_range_small_term_query.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range_field_disjunction_big_range_small_term_query.yaml new file mode 100644 index 00000000000..24cabf88754 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range_field_disjunction_big_range_small_term_query.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + LogicalFilter(condition=[OR(=($34, 'indigodagger'), SEARCH($28, Sarg[[1..100]]))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, aws.cloudwatch.log_stream, event], FILTER->OR(=($15, 'indigodagger'), SEARCH($13, Sarg[[1..100]])), LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"should":[{"term":{"aws.cloudwatch.log_stream":{"value":"indigodagger","boost":1.0}}},{"range":{"metrics.size":{"from":1.0,"to":100.0,"include_lower":true,"include_upper":true,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range_numeric.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range_numeric.yaml new file mode 100644 index 00000000000..cdf19c603a0 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range_numeric.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + LogicalFilter(condition=[SEARCH($28, Sarg[[20..200]])]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], FILTER->SEARCH($13, Sarg[[20..200]]), LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"range":{"metrics.size":{"from":20.0,"to":200.0,"include_lower":true,"include_upper":true,"boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range_with_asc_sort.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range_with_asc_sort.yaml new file mode 100644 index 00000000000..e0b91168f1f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range_with_asc_sort.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <=($17, TIMESTAMP('2023-01-13 00:00:00':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->SEARCH($7, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-13 00:00:00':VARCHAR]]:VARCHAR), SORT->[{ + "@timestamp" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-13T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range_with_desc_sort.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range_with_desc_sort.yaml new file mode 100644 index 00000000000..8af1fc7058d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range_with_desc_sort.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$7], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$17], dir0=[DESC-nulls-last], fetch=[10]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <=($17, TIMESTAMP('2023-01-13 00:00:00':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->SEARCH($7, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-13 00:00:00':VARCHAR]]:VARCHAR), SORT->[{ + "@timestamp" : { + "order" : "desc", + "missing" : "_last" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-13T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/scroll.yaml b/integ-test/src/test/resources/expectedOutput/calcite/scroll.yaml new file mode 100644 index 00000000000..59e68e48769 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/scroll.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/sort_keyword_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/calcite/sort_keyword_can_match_shortcut.yaml new file mode 100644 index 00000000000..501c35a492a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/sort_keyword_can_match_shortcut.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$25], dir0=[ASC-nulls-first], fetch=[10]) + LogicalFilter(condition=[query_string(MAP('query', 'process.name:kernel':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, meta.file, host, metrics, aws, event], FILTER->query_string(MAP('query', 'process.name:kernel':VARCHAR)), SORT->[{ + "meta.file" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"meta.file":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/sort_keyword_no_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/calcite/sort_keyword_no_can_match_shortcut.yaml new file mode 100644 index 00000000000..501c35a492a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/sort_keyword_no_can_match_shortcut.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$25], dir0=[ASC-nulls-first], fetch=[10]) + LogicalFilter(condition=[query_string(MAP('query', 'process.name:kernel':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, meta.file, host, metrics, aws, event], FILTER->query_string(MAP('query', 'process.name:kernel':VARCHAR)), SORT->[{ + "meta.file" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"meta.file":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_asc.yaml b/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_asc.yaml new file mode 100644 index 00000000000..cbbc5106ec6 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_asc.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$28], dir0=[ASC-nulls-first], fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], SORT->[{ + "metrics.size" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"metrics.size":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_asc_with_match.yaml b/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_asc_with_match.yaml new file mode 100644 index 00000000000..9aa906cc6ca --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_asc_with_match.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$28], dir0=[ASC-nulls-first], fetch=[10]) + LogicalFilter(condition=[query_string(MAP('query', 'log.file.path:\/var\/log\/messages\/solarshark':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], FILTER->query_string(MAP('query', 'log.file.path:\/var\/log\/messages\/solarshark':VARCHAR)), SORT->[{ + "metrics.size" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"log.file.path:\\/var\\/log\\/messages\\/solarshark","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"metrics.size":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_desc.yaml b/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_desc.yaml new file mode 100644 index 00000000000..3f059c7519f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_desc.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$28], dir0=[DESC-nulls-last], fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], SORT->[{ + "metrics.size" : { + "order" : "desc", + "missing" : "_last" + } + }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"metrics.size":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_desc_with_match.yaml b/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_desc_with_match.yaml new file mode 100644 index 00000000000..b52bb433722 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_desc_with_match.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$28], dir0=[DESC-nulls-last], fetch=[10]) + LogicalFilter(condition=[query_string(MAP('query', 'log.file.path:\/var\/log\/messages\/solarshark':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], FILTER->query_string(MAP('query', 'log.file.path:\/var\/log\/messages\/solarshark':VARCHAR)), SORT->[{ + "metrics.size" : { + "order" : "desc", + "missing" : "_last" + } + }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"log.file.path:\\/var\\/log\\/messages\\/solarshark","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"metrics.size":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/term.yaml b/integ-test/src/test/resources/expectedOutput/calcite/term.yaml new file mode 100644 index 00000000000..21c0d2d0e5d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/term.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + LogicalFilter(condition=[=($10, '/var/log/messages/birdknight')]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, log.file.path, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->=($3, '/var/log/messages/birdknight'), LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"term":{"log.file.path":{"value":"/var/log/messages/birdknight","boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/terms_significant_1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/terms_significant_1.yaml new file mode 100644 index 00000000000..2f3aab7b147 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/terms_significant_1.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(fetch=[10]) + LogicalProject(count()=[$2], aws.cloudwatch.log_stream=[$0], process.name=[$1]) + LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) + LogicalProject(aws.cloudwatch.log_stream=[$34], process.name=[$7]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[process.name, @timestamp, aws.cloudwatch.log_stream], FILTER->SEARCH($1, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), aws.cloudwatch.log_stream, process.name], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["process.name","@timestamp","aws.cloudwatch.log_stream"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"process.name":{"terms":{"field":"process.name","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/terms_significant_2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/terms_significant_2.yaml new file mode 100644 index 00000000000..cf04d9b8695 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/terms_significant_2.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(fetch=[10]) + LogicalProject(count()=[$2], process.name=[$0], aws.cloudwatch.log_stream=[$1]) + LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) + LogicalProject(process.name=[$7], aws.cloudwatch.log_stream=[$34]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[process.name, @timestamp, aws.cloudwatch.log_stream], FILTER->SEARCH($1, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), process.name, aws.cloudwatch.log_stream], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["process.name","@timestamp","aws.cloudwatch.log_stream"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/udf_geoip_in_agg_pushed.yaml b/integ-test/src/test/resources/expectedOutput/calcite/udf_geoip_in_agg_pushed.yaml new file mode 100644 index 00000000000..baf08f483a8 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/udf_geoip_in_agg_pushed.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], info.city=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(info.city=[ITEM(GEOIP('my-datasource':VARCHAR, $0), 'city')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), info.city], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"info.city":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAknsKICAiZmllbGRzIjogWwogICAgewogICAgICAidWR0IjogIkVYUFJfSVAiLAogICAgICAidHlwZSI6ICJPVEhFUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogImhvc3QiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQETnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJJVEVNIiwKICAgICJraW5kIjogIklURU0iLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiR0VPSVAiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAibXktZGF0YXNvdXJjZSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiTUFQIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAia2V5IjogewogICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgIH0sCiAgICAgICAgInZhbHVlIjogewogICAgICAgICAgInR5cGUiOiAiQU5ZIiwKICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAgICAgInNjYWxlIjogLTIxNDc0ODM2NDgKICAgICAgICB9CiAgICAgIH0sCiAgICAgICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAgICAgImR5bmFtaWMiOiBmYWxzZQogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAiY2l0eSIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiA0CiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AARob3N0fnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAACSVB4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/access_struct_subfield_with_item.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/access_struct_subfield_with_item.yaml new file mode 100644 index 00000000000..afd78ed8c22 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/access_struct_subfield_with_item.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(host=[$0], info=[GEOIP('dummy-datasource':VARCHAR, $0)], info.dummy_sub_field=[ITEM(GEOIP('dummy-datasource':VARCHAR, $0), 'dummy_sub_field')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..11=[{inputs}], expr#12=['dummy-datasource':VARCHAR], expr#13=[GEOIP($t12, $t0)], expr#14=['dummy_sub_field'], expr#15=[ITEM($t13, $t14)], host=[$t0], info=[$t13], info.dummy_sub_field=[$t15]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_cannot_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_cannot_push.yaml new file mode 100644 index 00000000000..f8fd80e1598 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_cannot_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg_age=[$1], age_range=[$0]) + LogicalAggregate(group=[{0}], avg_age=[AVG($1)]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, SEARCH($10, Sarg[[30..40]]), 'u40':VARCHAR, 'u100':VARCHAR)], age=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg_age=[$t8], age_range=[$t0]) + EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[Sarg[[30..40]]], expr#23=[SEARCH($t10, $t22)], expr#24=['u40':VARCHAR], expr#25=['u100':VARCHAR], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], age_range=[$t26], age=[$t10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_composite_cannot_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_composite_cannot_push.yaml new file mode 100644 index 00000000000..059caa2e2d2 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_composite_cannot_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg_balance=[$2], age_range=[$0], state=[$1]) + LogicalAggregate(group=[{0, 1}], avg_balance=[AVG($2)]) + LogicalProject(age_range=[CASE(<($10, 35), 'u35':VARCHAR, $11)], state=[$9], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg_balance=[$t9], age_range=[$t0], state=[$t1]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[35], expr#20=[<($t10, $t19)], expr#21=['u35':VARCHAR], expr#22=[CASE($t20, $t21, $t11)], age_range=[$t22], state=[$t9], balance=[$t7]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_num_res_cannot_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_num_res_cannot_push.yaml new file mode 100644 index 00000000000..46035abe925 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_num_res_cannot_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], age_range=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(age_range=[CASE(<($10, 30), 30, 100)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], age_range=[$t0]) + EnumerableAggregate(group=[{0}], count()=[COUNT()]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=[100], expr#22=[CASE($t20, $t19, $t21)], age_range=[$t22]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_count_push.yaml new file mode 100644 index 00000000000..43e27cd2d5d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_count_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg(balance)=[$3], count()=[$4], age_range=[$0], state=[$1], gender=[$2]) + LogicalAggregate(group=[{0, 1, 2}], avg(balance)=[AVG($3)], count()=[COUNT()]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, 'a30':VARCHAR)], state=[$9], gender=[$4], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t4, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t8, $t3)], expr#10=[CAST($t9):DOUBLE], expr#11=[/($t10, $t4)], avg(balance)=[$t11], count()=[$t5], age_range=[$t0], state=[$t1], gender=[$t2]) + EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($3)], agg#1=[COUNT($3)], count()=[COUNT()]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=['a30':VARCHAR], expr#23=[CASE($t20, $t21, $t22)], age_range=[$t23], state=[$t9], gender=[$t4], balance=[$t7]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_range_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_range_count_push.yaml new file mode 100644 index 00000000000..6dfa7cd65a3 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_range_count_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg_balance=[$3], age_range=[$0], balance_range=[$1], state=[$2]) + LogicalAggregate(group=[{0, 1, 2}], avg_balance=[AVG($3)]) + LogicalProject(age_range=[CASE(<($10, 35), 'u35':VARCHAR, 'a35':VARCHAR)], balance_range=[CASE(<($7, 20000), 'medium':VARCHAR, 'high':VARCHAR)], state=[$9], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t4, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t3)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t4)], avg_balance=[$t10], age_range=[$t0], balance_range=[$t1], state=[$t2]) + EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($3)], agg#1=[COUNT($3)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[35], expr#20=[<($t10, $t19)], expr#21=['u35':VARCHAR], expr#22=['a35':VARCHAR], expr#23=[CASE($t20, $t21, $t22)], expr#24=[20000], expr#25=[<($t7, $t24)], expr#26=['medium':VARCHAR], expr#27=['high':VARCHAR], expr#28=[CASE($t25, $t26, $t27)], age_range=[$t23], balance_range=[$t28], state=[$t9], balance=[$t7]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_date_range_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_date_range_push.yaml new file mode 100644 index 00000000000..f99713d9aaa --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_date_range_push.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg(value)=[$2], span(@timestamp,1h)=[$1], value_range=[$0]) + LogicalAggregate(group=[{0, 2}], avg(value)=[AVG($1)]) + LogicalProject(value_range=[$10], value=[$2], span(@timestamp,1h)=[SPAN($0, 1, 'h')]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9], value_range=[CASE(<($2, 7000), 'small':VARCHAR, 'large':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg(value)=[$t9], span(@timestamp,1h)=[$t1], value_range=[$t0]) + EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=[7000], expr#11=[<($t2, $t10)], expr#12=['small':VARCHAR], expr#13=['large':VARCHAR], expr#14=[CASE($t11, $t12, $t13)], expr#15=[1], expr#16=['h'], expr#17=[SPAN($t0, $t15, $t16)], expr#18=[IS NOT NULL($t0)], value_range=[$t14], value=[$t2], span(@timestamp,1h)=[$t17], $condition=[$t18]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_range_metric_push.yaml new file mode 100644 index 00000000000..41ed8ba61fc --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_range_metric_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg(balance)=[$2], state=[$0], age_range=[$1]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(state=[$9], age_range=[CASE(<($10, 30), 'u30':VARCHAR, 'a30':VARCHAR)], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg(balance)=[$t9], state=[$t0], age_range=[$t1]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=['a30':VARCHAR], expr#23=[CASE($t20, $t21, $t22)], state=[$t9], age_range=[$t23], balance=[$t7]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_count_push.yaml new file mode 100644 index 00000000000..67ad0f0fd07 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_count_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg(age)=[$1], age_range=[$0]) + LogicalAggregate(group=[{0}], avg(age)=[AVG($1)]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, SEARCH($10, Sarg[[30..40)]), 'u40':VARCHAR, 'u100':VARCHAR)], age=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg(age)=[$t8], age_range=[$t0]) + EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[Sarg[[30..40)]], expr#23=[SEARCH($t10, $t22)], expr#24=['u40':VARCHAR], expr#25=['u100':VARCHAR], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], age_range=[$t26], age=[$t10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_complex_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_complex_push.yaml new file mode 100644 index 00000000000..10ead7ad449 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_complex_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg(balance)=[$1], age_range=[$0]) + LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, SEARCH($10, Sarg[[35..40), [80..+∞)]), '30-40 or >=80':VARCHAR, null:NULL)], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg(balance)=[$t8], age_range=[$t0]) + EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[Sarg[[35..40), [80..+∞)]], expr#23=[SEARCH($t10, $t22)], expr#24=['30-40 or >=80':VARCHAR], expr#25=[null:NULL], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], age_range=[$t26], balance=[$t7]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_push.yaml new file mode 100644 index 00000000000..a81e208bdbf --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg_age=[$1], age_range=[$0]) + LogicalAggregate(group=[{0}], avg_age=[AVG($1)]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, <($10, 40), 'u40':VARCHAR, 'u100':VARCHAR)], age=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg_age=[$t8], age_range=[$t0]) + EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[40], expr#23=[<($t10, $t22)], expr#24=['u40':VARCHAR], expr#25=['u100':VARCHAR], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], age_range=[$t26], age=[$t10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_range_metric_push.yaml new file mode 100644 index 00000000000..404726f6083 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_range_metric_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg_balance=[$2], age_range=[$0], balance_range=[$1]) + LogicalAggregate(group=[{0, 1}], avg_balance=[AVG($2)]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, <($10, 40), 'u40':VARCHAR, 'u100':VARCHAR)], balance_range=[CASE(<($7, 20000), 'medium':VARCHAR, 'high':VARCHAR)], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg_balance=[$t9], age_range=[$t0], balance_range=[$t1]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[40], expr#23=[<($t10, $t22)], expr#24=['u40':VARCHAR], expr#25=['u100':VARCHAR], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], expr#27=[20000], expr#28=[<($t7, $t27)], expr#29=['medium':VARCHAR], expr#30=['high':VARCHAR], expr#31=[CASE($t28, $t29, $t30)], age_range=[$t26], balance_range=[$t31], balance=[$t7]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_group_merge.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_group_merge.yaml new file mode 100644 index 00000000000..a694c63b2ca --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_group_merge.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$4], age1=[$0], age2=[$1], age3=[$2], age=[$3]) + LogicalAggregate(group=[{0, 1, 2, 3}], count()=[COUNT()]) + LogicalProject(age1=[*($8, 10)], age2=[+($8, 10)], age3=[10], age=[$8]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10], expr#3=[*($t0, $t2)], expr#4=[+($t0, $t2)], count()=[$t1], age1=[$t3], age2=[$t4], age3=[$t2], age=[$t0]) + EnumerableAggregate(group=[{8}], count()=[COUNT()]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_script_timestamp_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_script_timestamp_push.json deleted file mode 100644 index e0dcbaaeb8d..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_script_timestamp_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], fetch=[3])\n LogicalProject(count()=[$1], t=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(t=[UNIX_TIMESTAMP($3)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], t=[$t0])\n EnumerableLimit(fetch=[3])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n EnumerableAggregate(group=[{0}], count()=[COUNT()])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[UNIX_TIMESTAMP($t3)], t=[$t19])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_script_timestamp_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_script_timestamp_push.yaml new file mode 100644 index 00000000000..d1119b4f557 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_script_timestamp_push.yaml @@ -0,0 +1,16 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], fetch=[3]) + LogicalProject(count()=[$1], t=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(t=[UNIX_TIMESTAMP($3)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], t=[$t0]) + EnumerableLimit(fetch=[3]) + EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first]) + EnumerableAggregate(group=[{0}], count()=[COUNT()]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[UNIX_TIMESTAMP($t3)], t=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_script_udt_arg_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_script_udt_arg_push.json deleted file mode 100644 index 39ec3279c63..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_script_udt_arg_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(t,1d)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(t,1d)=[SPAN($19, 1, 'd')])\n LogicalFilter(condition=[IS NOT NULL($19)])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], t=[DATE_ADD($3, 1:INTERVAL DAY)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], span(t,1d)=[$t0])\n EnumerableAggregate(group=[{0}], count()=[COUNT()])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1:INTERVAL DAY], expr#20=[DATE_ADD($t3, $t19)], expr#21=[1], expr#22=['d'], expr#23=[SPAN($t20, $t21, $t22)], expr#24=[IS NOT NULL($t20)], span(t,1d)=[$t23], $condition=[$t24])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_script_udt_arg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_script_udt_arg_push.yaml new file mode 100644 index 00000000000..b5a045807e0 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_script_udt_arg_push.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], span(t,1d)=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(span(t,1d)=[SPAN($19, 1, 'd')]) + LogicalFilter(condition=[IS NOT NULL($19)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], t=[DATE_ADD($3, 1:INTERVAL DAY)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], span(t,1d)=[$t0]) + EnumerableAggregate(group=[{0}], count()=[COUNT()]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1:INTERVAL DAY], expr#20=[DATE_ADD($t3, $t19)], expr#21=[1], expr#22=['d'], expr#23=[SPAN($t20, $t21, $t22)], expr#24=[IS NOT NULL($t20)], span(t,1d)=[$t23], $condition=[$t24]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.json deleted file mode 100644 index 889e36d69a7..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(sum=[$2], len=[$0], gender=[$1])\n LogicalAggregate(group=[{0, 1}], sum=[SUM($2)])\n LogicalProject(len=[CHAR_LENGTH($4)], gender=[$4], $f3=[+($7, 100)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..2=[{inputs}], sum=[$t2], len=[$t0], gender=[$t1])\n EnumerableAggregate(group=[{0, 1}], sum=[SUM($2)])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CHAR_LENGTH($t4)], expr#20=[100], expr#21=[+($t7, $t20)], len=[$t19], gender=[$t4], $f3=[$t21])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml new file mode 100644 index 00000000000..285d0b221e1 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(sum=[$2], len=[$0], gender=[$1]) + LogicalAggregate(group=[{0, 1}], sum=[SUM($2)]) + LogicalProject(len=[CHAR_LENGTH($4)], gender=[$4], $f3=[+($7, 100)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[100], expr#8=[*($t2, $t7)], expr#9=[+($t6, $t8)], expr#10=[CHAR_LENGTH($t0)], sum=[$t9], len=[$t10], gender=[$t0]) + EnumerableAggregate(group=[{4}], sum_SUM=[$SUM0($7)], agg#1=[COUNT($7)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest.json deleted file mode 100644 index ad2df524707..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(earliest_message=[$1], latest_message=[$2], server=[$0])\n LogicalAggregate(group=[{0}], earliest_message=[ARG_MIN($1, $2)], latest_message=[ARG_MAX($1, $2)])\n LogicalProject(server=[$1], message=[$3], @timestamp=[$2])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..2=[{inputs}], earliest_message=[$t1], latest_message=[$t2], server=[$t0])\n EnumerableAggregate(group=[{1}], earliest_message=[ARG_MIN($3, $2)], latest_message=[ARG_MAX($3, $2)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest.yaml new file mode 100644 index 00000000000..32496a5b10e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(earliest_message=[$1], latest_message=[$2], server=[$0]) + LogicalAggregate(group=[{0}], earliest_message=[ARG_MIN($1, $2)], latest_message=[ARG_MAX($1, $2)]) + LogicalProject(server=[$1], message=[$3], @timestamp=[$2]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], earliest_message=[$t1], latest_message=[$t2], server=[$t0]) + EnumerableAggregate(group=[{1}], earliest_message=[ARG_MIN($3, $2)], latest_message=[ARG_MAX($3, $2)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest_custom_time.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest_custom_time.json deleted file mode 100644 index ef5ebfe9e1c..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest_custom_time.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(earliest_message=[$1], latest_message=[$2], level=[$0])\n LogicalAggregate(group=[{0}], earliest_message=[ARG_MIN($1, $2)], latest_message=[ARG_MAX($1, $2)])\n LogicalProject(level=[$4], message=[$3], created_at=[$0])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..2=[{inputs}], earliest_message=[$t1], latest_message=[$t2], level=[$t0])\n EnumerableAggregate(group=[{4}], earliest_message=[ARG_MIN($3, $0)], latest_message=[ARG_MAX($3, $0)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest_custom_time.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest_custom_time.yaml new file mode 100644 index 00000000000..d9e4320bc4a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest_custom_time.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(earliest_message=[$1], latest_message=[$2], level=[$0]) + LogicalAggregate(group=[{0}], earliest_message=[ARG_MIN($1, $2)], latest_message=[ARG_MAX($1, $2)]) + LogicalProject(level=[$4], message=[$3], created_at=[$0]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], earliest_message=[$t1], latest_message=[$t2], level=[$t0]) + EnumerableAggregate(group=[{4}], earliest_message=[ARG_MIN($3, $0)], latest_message=[ARG_MAX($3, $0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_exists_correlated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_exists_correlated_subquery.yaml new file mode 100644 index 00000000000..400bd549ee8 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_exists_correlated_subquery.yaml @@ -0,0 +1,25 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(id=[$2], name=[$0], salary=[$4]) + LogicalSort(sort0=[$4], dir0=[DESC-nulls-last]) + LogicalFilter(condition=[EXISTS({ + LogicalProject(name=[$0], uid=[$1], occupation=[$2], department=[$3]) + LogicalFilter(condition=[=($cor0.id, $1)]) + LogicalSystemLimit(fetch=[10000], type=[SUBSEARCH_MAXOUT]) + LogicalFilter(condition=[=($0, 'Tom')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) + })], variablesSet=[[$cor0]]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) + EnumerableSort(sort0=[$2], dir0=[DESC-nulls-last]) + EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{1}]) + EnumerableCalc(expr#0..10=[{inputs}], name=[$t0], id=[$t2], salary=[$t4]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + EnumerableAggregate(group=[{0}]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], expr#3=[$cor0], expr#4=[$t3.id], expr#5=[=($t4, $t1)], i=[$t2], $condition=[$t5]) + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=['Tom':VARCHAR], expr#11=[=($t0, $t10)], proj#0..1=[{exprs}], $condition=[$t11]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_exists_uncorrelated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_exists_uncorrelated_subquery.yaml new file mode 100644 index 00000000000..3b4e34539c7 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_exists_uncorrelated_subquery.yaml @@ -0,0 +1,24 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(id=[$2], name=[$0], salary=[$4]) + LogicalSort(sort0=[$4], dir0=[DESC-nulls-last]) + LogicalFilter(condition=[EXISTS({ + LogicalSystemLimit(fetch=[10000], type=[SUBSEARCH_MAXOUT]) + LogicalProject(name=[$0], uid=[$1], occupation=[$2], department=[$3]) + LogicalFilter(condition=[=($0, 'Tom')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) + })], variablesSet=[[$cor0]]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) + EnumerableSort(sort0=[$2], dir0=[DESC-nulls-last]) + EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) + EnumerableCalc(expr#0..10=[{inputs}], name=[$t0], id=[$t2], salary=[$t4]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + EnumerableAggregate(group=[{0}]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=[true], i=[$t10]) + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=['Tom':VARCHAR], expr#11=[=($t0, $t10)], proj#0..9=[{exprs}], $condition=[$t11]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_fillnull_value_syntax.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_fillnull_value_syntax.yaml new file mode 100644 index 00000000000..740047e9805 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_fillnull_value_syntax.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[COALESCE($8, 0)], balance=[COALESCE($3, 0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[0], expr#18=[COALESCE($t8, $t17)], expr#19=[COALESCE($t3, $t17)], age=[$t18], balance=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_function_script_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_function_script_push.json deleted file mode 100644 index 6640d6c214a..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_function_script_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(firstname=[$1], age=[$8])\n LogicalFilter(condition=[AND(=(CHAR_LENGTH($1), 5), =(ABS($8), 32), =($3, 39225))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[CHAR_LENGTH($t1)], expr#18=[5], expr#19=[=($t17, $t18)], expr#20=[ABS($t8)], expr#21=[32], expr#22=[=($t20, $t21)], expr#23=[39225], expr#24=[=($t3, $t23)], expr#25=[AND($t19, $t22, $t24)], firstname=[$t1], age=[$t8], $condition=[$t25])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_function_script_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_function_script_push.yaml new file mode 100644 index 00000000000..0cc53c7287f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_function_script_push.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(firstname=[$1], age=[$8]) + LogicalFilter(condition=[AND(=(CHAR_LENGTH($1), 5), =(ABS($8), 32), =($3, 39225))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[CHAR_LENGTH($t1)], expr#18=[5], expr#19=[=($t17, $t18)], expr#20=[ABS($t8)], expr#21=[32], expr#22=[=($t20, $t21)], expr#23=[39225], expr#24=[=($t3, $t23)], expr#25=[AND($t19, $t22, $t24)], firstname=[$t1], age=[$t8], $condition=[$t25]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_push.json deleted file mode 100644 index cadaa9938bc..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(firstname=[$1], age=[$8])\n LogicalFilter(condition=[AND(=($1, 'Amber'), =(-($8, 2), 30))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=['Amber':VARCHAR], expr#18=[=($t1, $t17)], expr#19=[2], expr#20=[-($t8, $t19)], expr#21=[30], expr#22=[=($t20, $t21)], expr#23=[AND($t18, $t22)], firstname=[$t1], age=[$t8], $condition=[$t23])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_push.yaml new file mode 100644 index 00000000000..90492abbaf8 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_push.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(firstname=[$1], age=[$8]) + LogicalFilter(condition=[AND(=($1, 'Amber'), =(-($8, 2), 30))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['Amber':VARCHAR], expr#18=[=($t1, $t17)], expr#19=[2], expr#20=[-($t8, $t19)], expr#21=[30], expr#22=[=($t20, $t21)], expr#23=[AND($t18, $t22)], firstname=[$t1], age=[$t8], $condition=[$t23]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_first_last.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_first_last.json deleted file mode 100644 index cd87821aed6..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_first_last.json +++ /dev/null @@ -1 +0,0 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(first_name=[$1], last_name=[$2], gender=[$0])\n LogicalAggregate(group=[{0}], first_name=[FIRST($1)], last_name=[LAST($1)])\n LogicalProject(gender=[$4], firstname=[$1])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..2=[{inputs}], first_name=[$t1], last_name=[$t2], gender=[$t0])\n EnumerableAggregate(group=[{4}], first_name=[FIRST($1)], last_name=[LAST($1)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n"}} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_first_last.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_first_last.yaml new file mode 100644 index 00000000000..ee569fbbf3a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_first_last.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(first_name=[$1], last_name=[$2], gender=[$0]) + LogicalAggregate(group=[{0}], first_name=[FIRST($1)], last_name=[LAST($1)]) + LogicalProject(gender=[$4], firstname=[$1]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], first_name=[$t1], last_name=[$t2], gender=[$t0]) + EnumerableAggregate(group=[{4}], first_name=[FIRST($1)], last_name=[LAST($1)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_in_correlated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_in_correlated_subquery.yaml new file mode 100644 index 00000000000..cb17a67d1cd --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_in_correlated_subquery.yaml @@ -0,0 +1,26 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(id=[$2], name=[$0], salary=[$4]) + LogicalSort(sort0=[$4], dir0=[DESC-nulls-last]) + LogicalFilter(condition=[IN($0, { + LogicalProject(name=[$0]) + LogicalFilter(condition=[=($cor0.id, $1)]) + LogicalSystemLimit(fetch=[10000], type=[SUBSEARCH_MAXOUT]) + LogicalFilter(condition=[=($0, 'Tom')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) + })], variablesSet=[[$cor0]]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + physical: | + EnumerableCalc(expr#0..3=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$2], dir0=[DESC-nulls-last]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[=($t0, $t3)], proj#0..3=[{exprs}], $condition=[$t4]) + EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{1}]) + EnumerableCalc(expr#0..10=[{inputs}], name=[$t0], id=[$t2], salary=[$t4]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + EnumerableAggregate(group=[{0}]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[$cor0], expr#3=[$t2.id], expr#4=[=($t3, $t1)], proj#0..1=[{exprs}], $condition=[$t4]) + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=['Tom':VARCHAR], expr#11=[=($t0, $t10)], proj#0..1=[{exprs}], $condition=[$t11]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_in_uncorrelated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_in_uncorrelated_subquery.yaml new file mode 100644 index 00000000000..e94a46d70d8 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_in_uncorrelated_subquery.yaml @@ -0,0 +1,20 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(id=[$2], name=[$0], salary=[$4]) + LogicalSort(sort0=[$4], dir0=[DESC-nulls-last]) + LogicalFilter(condition=[IN($2, { + LogicalSystemLimit(fetch=[10000], type=[SUBSEARCH_MAXOUT]) + LogicalProject(uid=[$1]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) + })], variablesSet=[[$cor0]]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) + EnumerableSort(sort0=[$2], dir0=[DESC-nulls-last]) + EnumerableHashJoin(condition=[=($1, $4)], joinType=[semi]) + EnumerableCalc(expr#0..10=[{inputs}], name=[$t0], id=[$t2], salary=[$t4]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isblank.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isblank.json deleted file mode 100644 index f70401bca2b..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isblank.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10])\n LogicalFilter(condition=[OR(IS NULL($1), IS EMPTY(TRIM(FLAG(BOTH), ' ', $1)))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NULL($t1)], expr#18=[FLAG(BOTH)], expr#19=[' '], expr#20=[TRIM($t18, $t19, $t1)], expr#21=[IS EMPTY($t20)], expr#22=[OR($t17, $t21)], proj#0..10=[{exprs}], $condition=[$t22])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isblank.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isblank.yaml new file mode 100644 index 00000000000..887fd96408b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isblank.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalFilter(condition=[OR(IS NULL($1), IS EMPTY(TRIM(FLAG(BOTH), ' ', $1)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NULL($t1)], expr#18=[FLAG(BOTH)], expr#19=[' '], expr#20=[TRIM($t18, $t19, $t1)], expr#21=[IS EMPTY($t20)], expr#22=[OR($t17, $t21)], proj#0..10=[{exprs}], $condition=[$t22]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty.json deleted file mode 100644 index ce1187d0a0c..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10])\n LogicalFilter(condition=[OR(IS NULL($1), IS EMPTY($1))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NULL($t1)], expr#18=[IS EMPTY($t1)], expr#19=[OR($t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty.yaml new file mode 100644 index 00000000000..6115f98e23f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalFilter(condition=[OR(IS NULL($1), IS EMPTY($1))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NULL($t1)], expr#18=[IS EMPTY($t1)], expr#19=[OR($t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty_or_others.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty_or_others.json deleted file mode 100644 index 4df3f576995..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty_or_others.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10])\n LogicalFilter(condition=[OR(=($4, 'M'), IS NULL($1), IS EMPTY($1))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NULL($t1)], expr#18=['M'], expr#19=[=($t4, $t18)], expr#20=[IS EMPTY($t1)], expr#21=[OR($t17, $t19, $t20)], proj#0..10=[{exprs}], $condition=[$t21])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty_or_others.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty_or_others.yaml new file mode 100644 index 00000000000..7f43f48dc57 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_isempty_or_others.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalFilter(condition=[OR(=($4, 'M'), IS NULL($1), IS EMPTY($1))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NULL($t1)], expr#18=['M'], expr#19=[=($t4, $t18)], expr#20=[IS EMPTY($t1)], expr#21=[OR($t17, $t19, $t20)], proj#0..10=[{exprs}], $condition=[$t21]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_join_with_fields.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_join_with_fields.json deleted file mode 100644 index 21cbcfab737..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_join_with_fields.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$13], firstname=[$14], address=[$15], birthdate=[$16], gender=[$17], city=[$18], lastname=[$19], balance=[$20], employer=[$21], state=[$22], age=[$23], email=[$24], male=[$25])\n LogicalJoin(condition=[=($0, $13)], joinType=[left])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableCalc(expr#0..13=[{inputs}], account_number=[$t1], firstname=[$t2], address=[$t3], birthdate=[$t4], gender=[$t5], city=[$t6], lastname=[$t7], balance=[$t8], employer=[$t9], state=[$t10], age=[$t11], email=[$t12], male=[$t13])\n EnumerableLimit(fetch=[10000])\n EnumerableMergeJoin(condition=[=($0, $1)], joinType=[left])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableCalc(expr#0..18=[{inputs}], account_number=[$t0])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableCalc(expr#0..18=[{inputs}], proj#0..12=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_join_with_fields.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_join_with_fields.yaml new file mode 100644 index 00000000000..bf397010f6b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_join_with_fields.yaml @@ -0,0 +1,21 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$13], firstname=[$14], address=[$15], birthdate=[$16], gender=[$17], city=[$18], lastname=[$19], balance=[$20], employer=[$21], state=[$22], age=[$23], email=[$24], male=[$25]) + LogicalJoin(condition=[=($0, $13)], joinType=[left]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableCalc(expr#0..13=[{inputs}], account_number=[$t1], firstname=[$t2], address=[$t3], birthdate=[$t4], gender=[$t5], city=[$t6], lastname=[$t7], balance=[$t8], employer=[$t9], state=[$t10], age=[$t11], email=[$t12], male=[$t13]) + EnumerableLimit(fetch=[10000]) + EnumerableMergeJoin(condition=[=($0, $1)], joinType=[left]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..18=[{inputs}], account_number=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableLimit(fetch=[50000]) + EnumerableCalc(expr#0..18=[{inputs}], proj#0..12=[{exprs}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function.json deleted file mode 100644 index 59162db88b9..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10])\n LogicalFilter(condition=[ILIKE($1, '%mbe%':VARCHAR, '\\')])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%mbe%':VARCHAR], expr#18=['\\'], expr#19=[ILIKE($t1, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function.yaml new file mode 100644 index 00000000000..f8b576cb814 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalFilter(condition=[ILIKE($1, '%mbe%', '\')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%mbe%'], expr#18=['\'], expr#19=[ILIKE($t1, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_max_string_field.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_max_string_field.json deleted file mode 100644 index 8a84763e33c..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_max_string_field.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], max(firstname)=[MAX($0)])\n LogicalProject(firstname=[$1])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableAggregate(group=[{}], max(firstname)=[MAX($1)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_max_string_field.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_max_string_field.yaml new file mode 100644 index 00000000000..fe8927692ab --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_max_string_field.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], max(firstname)=[MAX($0)]) + LogicalProject(firstname=[$1]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableAggregate(group=[{}], max(firstname)=[MAX($1)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_merge_join_sort_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_merge_join_sort_push.json deleted file mode 100644 index 084817d2f7d..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_merge_join_sort_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], r.account_number=[$13], r.firstname=[$14], r.address=[$15], r.birthdate=[$16], r.gender=[$17], r.city=[$18], r.lastname=[$19], r.balance=[$20], r.employer=[$21], r.state=[$22], r.age=[$23], r.email=[$24], r.male=[$25])\n LogicalJoin(condition=[=($0, $13)], joinType=[inner])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableMergeJoin(condition=[=($0, $13)], joinType=[inner])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableCalc(expr#0..18=[{inputs}], proj#0..12=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableCalc(expr#0..18=[{inputs}], proj#0..12=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_merge_join_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_merge_join_sort_push.yaml new file mode 100644 index 00000000000..843fa505511 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_merge_join_sort_push.yaml @@ -0,0 +1,20 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], r.account_number=[$13], r.firstname=[$14], r.address=[$15], r.birthdate=[$16], r.gender=[$17], r.city=[$18], r.lastname=[$19], r.balance=[$20], r.employer=[$21], r.state=[$22], r.age=[$23], r.email=[$24], r.male=[$25]) + LogicalJoin(condition=[=($0, $13)], joinType=[inner]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableMergeJoin(condition=[=($0, $13)], joinType=[inner]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..18=[{inputs}], proj#0..12=[{exprs}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableLimit(fetch=[50000]) + EnumerableCalc(expr#0..18=[{inputs}], proj#0..12=[{exprs}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_min_string_field.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_min_string_field.json deleted file mode 100644 index 320b519c442..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_min_string_field.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], min(firstname)=[MIN($0)])\n LogicalProject(firstname=[$1])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableAggregate(group=[{}], min(firstname)=[MIN($1)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_min_string_field.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_min_string_field.yaml new file mode 100644 index 00000000000..027ecd556d9 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_min_string_field.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], min(firstname)=[MIN($0)]) + LogicalProject(firstname=[$1]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableAggregate(group=[{}], min(firstname)=[MIN($1)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multisearch_basic.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multisearch_basic.yaml new file mode 100644 index 00000000000..4910dc0a253 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multisearch_basic.yaml @@ -0,0 +1,22 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count=[$1], age_group=[$0]) + LogicalAggregate(group=[{0}], count=[COUNT()]) + LogicalProject(age_group=[$11]) + LogicalUnion(all=[true]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], age_group=['young':VARCHAR]) + LogicalFilter(condition=[<($8, 30)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], age_group=['adult':VARCHAR]) + LogicalFilter(condition=[>=($8, 30)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], count=[$t1], age_group=[$t0]) + EnumerableAggregate(group=[{0}], count=[COUNT()]) + EnumerableUnion(all=[true]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['young':VARCHAR], expr#18=[30], expr#19=[<($t8, $t18)], age_group=[$t17], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['adult':VARCHAR], expr#18=[30], expr#19=[>=($t8, $t18)], age_group=[$t17], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multisearch_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multisearch_timestamp.yaml new file mode 100644 index 00000000000..64b0ff25050 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multisearch_timestamp.yaml @@ -0,0 +1,25 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC], fetch=[5]) + LogicalUnion(all=[true]) + LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) + LogicalFilter(condition=[SEARCH($1, Sarg['A':VARCHAR, 'B':VARCHAR]:VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) + LogicalFilter(condition=[SEARCH($1, Sarg['E':VARCHAR, 'F':VARCHAR]:VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data2]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableLimit(fetch=[5]) + EnumerableMergeUnion(all=[true]) + EnumerableCalc(expr#0..9=[{inputs}], proj#0..3=[{exprs}]) + EnumerableLimit(fetch=[5]) + EnumerableSort(sort0=[$0], dir0=[DESC]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=[Sarg['A':VARCHAR, 'B':VARCHAR]:VARCHAR], expr#11=[SEARCH($t1, $t10)], proj#0..9=[{exprs}], $condition=[$t11]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + EnumerableCalc(expr#0..9=[{inputs}], proj#0..3=[{exprs}]) + EnumerableLimit(fetch=[5]) + EnumerableSort(sort0=[$0], dir0=[DESC]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=[Sarg['E':VARCHAR, 'F':VARCHAR]:VARCHAR], expr#11=[SEARCH($t1, $t10)], proj#0..9=[{exprs}], $condition=[$t11]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data2]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rare_usenull_false.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rare_usenull_false.yaml new file mode 100644 index 00000000000..49a464287ff --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rare_usenull_false.yaml @@ -0,0 +1,17 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(gender=[$0], state=[$1], count=[$2]) + LogicalFilter(condition=[<=($3, 2)]) + LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2)]) + LogicalAggregate(group=[{0, 1}], count=[COUNT()]) + LogicalProject(gender=[$4], state=[$7]) + LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[2], expr#5=[<=($t3, $t4)], proj#0..2=[{exprs}], $condition=[$t5]) + EnumerableWindow(window#0=[window(partition {0} order by [2] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableAggregate(group=[{4, 7}], count=[COUNT()]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NOT NULL($t4)], expr#18=[IS NOT NULL($t7)], expr#19=[AND($t17, $t18)], proj#0..16=[{exprs}], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rare_usenull_true.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rare_usenull_true.yaml new file mode 100644 index 00000000000..d7b401feb17 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rare_usenull_true.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(gender=[$0], state=[$1], count=[$2]) + LogicalFilter(condition=[<=($3, 2)]) + LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2)]) + LogicalAggregate(group=[{0, 1}], count=[COUNT()]) + LogicalProject(gender=[$4], state=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[2], expr#5=[<=($t3, $t4)], proj#0..2=[{exprs}], $condition=[$t5]) + EnumerableWindow(window#0=[window(partition {0} order by [2] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableAggregate(group=[{4, 7}], count=[COUNT()]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_replace_command.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_replace_command.yaml new file mode 100644 index 00000000000..bbebbf37f80 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_replace_command.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(state=[REPLACE($7, 'IL':VARCHAR, 'Illinois':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['IL':VARCHAR], expr#18=['Illinois':VARCHAR], expr#19=[REPLACE($t7, $t17, $t18)], state=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_replace_wildcard.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_replace_wildcard.yaml new file mode 100644 index 00000000000..194f680adf2 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_replace_wildcard.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(state=[REGEXP_REPLACE($7, '^\Q\E(.*?)\QL\E$':VARCHAR, 'STATE_IL':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['^\Q\E(.*?)\QL\E$':VARCHAR], expr#18=['STATE_IL':VARCHAR], expr#19=[REGEXP_REPLACE($t7, $t17, $t18)], state=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rex.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rex.yaml index 00dc5e4ef91..56fd60a0abe 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rex.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rex.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], initial=[$17]) LogicalSort(fetch=[5]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], initial=[REX_EXTRACT($10, '(?^[A-Z])', 1)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], initial=[REX_EXTRACT($10, '(?^[A-Z])', 'initial')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=['(?^[A-Z])'], expr#18=[1], expr#19=[REX_EXTRACT($t10, $t17, $t18)], proj#0..10=[{exprs}], initial=[$t19]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['(?^[A-Z])'], expr#18=['initial'], expr#19=[REX_EXTRACT($t10, $t17, $t18)], proj#0..10=[{exprs}], initial=[$t19]) EnumerableLimit(fetch=[5]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sarg_filter_push_time_range.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sarg_filter_push_time_range.json deleted file mode 100644 index 3b953902117..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sarg_filter_push_time_range.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12])\n LogicalFilter(condition=[AND(>=($3, TIMESTAMP('2016-12-08 00:00:00.000000000':VARCHAR)), <($3, TIMESTAMP('2018-11-09 00:00:00.000000000':VARCHAR)))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[Sarg[['2016-12-08 00:00:00':VARCHAR..'2018-11-09 00:00:00':VARCHAR)]:VARCHAR], expr#20=[SEARCH($t3, $t19)], proj#0..12=[{exprs}], $condition=[$t20])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sarg_filter_push_time_range.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sarg_filter_push_time_range.yaml new file mode 100644 index 00000000000..d6d96c9e057 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sarg_filter_push_time_range.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + LogicalFilter(condition=[AND(>=($3, TIMESTAMP('2016-12-08 00:00:00.000000000':VARCHAR)), <($3, TIMESTAMP('2018-11-09 00:00:00.000000000':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[Sarg[['2016-12-08 00:00:00':VARCHAR..'2018-11-09 00:00:00':VARCHAR)]:VARCHAR], expr#20=[SEARCH($t3, $t19)], proj#0..12=[{exprs}], $condition=[$t20]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_correlated_subquery_in_select.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_correlated_subquery_in_select.yaml new file mode 100644 index 00000000000..5e76c380ff2 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_correlated_subquery_in_select.yaml @@ -0,0 +1,24 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(variablesSet=[[$cor0]], id=[$2], name=[$0], count_dept=[$SCALAR_QUERY({ + LogicalAggregate(group=[{}], count(name)=[COUNT($0)]) + LogicalProject(name=[$0]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalFilter(condition=[=($cor0.id, $1)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) + })]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + physical: | + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[0:BIGINT], expr#6=[CASE($t4, $t5, $t3)], id=[$t1], name=[$t0], count_dept=[$t6]) + EnumerableLimit(fetch=[10000]) + EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($1, $2)], joinType=[left]) + EnumerableCalc(expr#0..10=[{inputs}], name=[$t0], id=[$t2]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t2)], expr#4=[0], expr#5=[CASE($t3, $t2, $t4)], uid=[$t0], count(name)=[$t5]) + EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + EnumerableAggregate(group=[{2}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + EnumerableAggregate(group=[{1}], count(name)=[COUNT($0)]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=[IS NOT NULL($t1)], expr#11=[IS NOT NULL($t0)], expr#12=[AND($t10, $t11)], proj#0..9=[{exprs}], $condition=[$t12]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_correlated_subquery_in_where.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_correlated_subquery_in_where.yaml new file mode 100644 index 00000000000..ed28cb93b50 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_correlated_subquery_in_where.yaml @@ -0,0 +1,21 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(id=[$2], name=[$0]) + LogicalFilter(condition=[=($2, $SCALAR_QUERY({ + LogicalAggregate(group=[{}], max(uid)=[MAX($0)]) + LogicalProject(uid=[$1]) + LogicalFilter(condition=[=($cor0.id, $1)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) + }))], variablesSet=[[$cor0]]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], id=[$t1], name=[$t0]) + EnumerableHashJoin(condition=[=($1, $2)], joinType=[semi]) + EnumerableCalc(expr#0..10=[{inputs}], name=[$t0], id=[$t2]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[=($t0, $t1)], proj#0..1=[{exprs}], $condition=[$t2]) + EnumerableAggregate(group=[{1}], max(uid)=[MAX($1)]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=[IS NOT NULL($t1)], proj#0..9=[{exprs}], $condition=[$t10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_uncorrelated_subquery_in_select.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_uncorrelated_subquery_in_select.yaml new file mode 100644 index 00000000000..a229cdc7bc0 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_uncorrelated_subquery_in_select.yaml @@ -0,0 +1,18 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(variablesSet=[[$cor0]], name=[$0], count_dept=[$SCALAR_QUERY({ + LogicalAggregate(group=[{}], count(name)=[COUNT($0)]) + LogicalProject(name=[$0]) + LogicalFilter(condition=[IS NOT NULL($0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) + })]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableNestedLoopJoin(condition=[true], joinType=[left]) + EnumerableCalc(expr#0..10=[{inputs}], name=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + EnumerableAggregate(group=[{}], count(name)=[COUNT($0)]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=[IS NOT NULL($t0)], proj#0..9=[{exprs}], $condition=[$t10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_uncorrelated_subquery_in_where.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_uncorrelated_subquery_in_where.yaml new file mode 100644 index 00000000000..ba13359c44d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_uncorrelated_subquery_in_where.yaml @@ -0,0 +1,20 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(name=[$0]) + LogicalFilter(condition=[>($2, +($SCALAR_QUERY({ + LogicalAggregate(group=[{}], count(name)=[COUNT($0)]) + LogicalProject(name=[$0]) + LogicalFilter(condition=[IS NOT NULL($0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) + }), 999))], variablesSet=[[$cor0]]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], name=[$t0]) + EnumerableNestedLoopJoin(condition=[>($1, +($2, 999))], joinType=[inner]) + EnumerableCalc(expr#0..10=[{inputs}], name=[$t0], id=[$t2]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + EnumerableAggregate(group=[{}], count(name)=[COUNT($0)]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=[IS NOT NULL($t0)], proj#0..9=[{exprs}], $condition=[$t10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_dc.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_dc.yaml new file mode 100644 index 00000000000..6ffa5ad304c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_dc.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], distinct_states=[DISTINCT_COUNT_APPROX($7) OVER (ROWS UNBOUNDED PRECEDING)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..17=[{inputs}], proj#0..10=[{exprs}], $11=[$t17]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [DISTINCT_COUNT_APPROX($7)])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_distinct_count.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_distinct_count.yaml new file mode 100644 index 00000000000..550cf0ea9cb --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_distinct_count.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], distinct_states=[$18]) + LogicalSort(sort0=[$17], dir0=[ASC]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], distinct_states=[DISTINCT_COUNT_APPROX($7) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..18=[{inputs}], proj#0..10=[{exprs}], distinct_states=[$t18]) + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$17], dir0=[ASC]) + EnumerableWindow(window#0=[window(partition {4} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [DISTINCT_COUNT_APPROX($7)])]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest.yaml new file mode 100644 index 00000000000..c37fae48771 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[$12], latest_message=[$13]) + LogicalSort(sort0=[$11], dir0=[ASC]) + LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], _id=[$5], _index=[$6], _score=[$7], _maxscore=[$8], _sort=[$9], _routing=[$10], __stream_seq__=[$11], earliest_message=[ARG_MIN($3, $2) OVER (PARTITION BY $1 ROWS UNBOUNDED PRECEDING)], latest_message=[ARG_MAX($3, $2) OVER (PARTITION BY $1 ROWS UNBOUNDED PRECEDING)]) + LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], _id=[$5], _index=[$6], _score=[$7], _maxscore=[$8], _sort=[$9], _routing=[$10], __stream_seq__=[ROW_NUMBER() OVER ()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) + physical: | + EnumerableCalc(expr#0..13=[{inputs}], proj#0..4=[{exprs}], earliest_message=[$t12], latest_message=[$t13]) + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$11], dir0=[ASC]) + EnumerableWindow(window#0=[window(partition {1} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ARG_MIN($3, $2), ARG_MAX($3, $2)])]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest_custom_time.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest_custom_time.yaml new file mode 100644 index 00000000000..b85e4b6b7bb --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest_custom_time.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[$12], latest_message=[$13]) + LogicalSort(sort0=[$11], dir0=[ASC]) + LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], _id=[$5], _index=[$6], _score=[$7], _maxscore=[$8], _sort=[$9], _routing=[$10], __stream_seq__=[$11], earliest_message=[ARG_MIN($3, $0) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING)], latest_message=[ARG_MAX($3, $0) OVER (PARTITION BY $4 ROWS UNBOUNDED PRECEDING)]) + LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], _id=[$5], _index=[$6], _score=[$7], _maxscore=[$8], _sort=[$9], _routing=[$10], __stream_seq__=[ROW_NUMBER() OVER ()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) + physical: | + EnumerableCalc(expr#0..13=[{inputs}], proj#0..4=[{exprs}], earliest_message=[$t12], latest_message=[$t13]) + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$11], dir0=[ASC]) + EnumerableWindow(window#0=[window(partition {4} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ARG_MIN($3, $0), ARG_MAX($3, $0)])]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest_no_group.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest_no_group.yaml new file mode 100644 index 00000000000..79dcbca7555 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest_no_group.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[ARG_MIN($3, $2) OVER (ROWS UNBOUNDED PRECEDING)], latest_message=[ARG_MAX($3, $2) OVER (ROWS UNBOUNDED PRECEDING)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..12=[{inputs}], proj#0..4=[{exprs}], $5=[$t11], $6=[$t12]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ARG_MIN($3, $2), ARG_MAX($3, $2)])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml new file mode 100644 index 00000000000..3ac52e02f55 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml @@ -0,0 +1,30 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$18]) + LogicalSort(sort0=[$17], dir0=[ASC]) + LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{4, 17}]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalAggregate(group=[{}], avg_age=[AVG($8)]) + LogicalFilter(condition=[AND(>=($17, -($cor0.__stream_seq__, 1)), <=($17, $cor0.__stream_seq__), =($4, $cor0.gender))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t16]) + EnumerableLimit(fetch=[10000]) + EnumerableHashJoin(condition=[AND(=($4, $13), =($11, $14), =($12, $15))], joinType=[left]) + EnumerableSort(sort0=[$11], dir0=[ASC]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[1], expr#19=[-($t17, $t18)], proj#0..10=[{exprs}], __stream_seq__=[$t17], $f12=[$t19]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t4, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t3)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t4)], proj#0..2=[{exprs}], avg_age=[$t10]) + EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($4)], agg#1=[COUNT($4)]) + EnumerableHashJoin(condition=[AND(=($0, $3), >=($5, $2), <=($5, $1))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2}]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[1], expr#19=[-($t17, $t18)], gender=[$t4], __stream_seq__=[$t17], $f12=[$t19]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + EnumerableCalc(expr#0..17=[{inputs}], gender=[$t4], age=[$t8], $2=[$t17]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml new file mode 100644 index 00000000000..be28e9b1d8c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml @@ -0,0 +1,38 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$21]) + LogicalSort(sort0=[$17], dir0=[ASC]) + LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{4, 17, 20}]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), COALESCE(SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalAggregate(group=[{}], avg_age=[AVG($8)]) + LogicalFilter(condition=[AND(<($17, $cor0.__stream_seq__), =($20, $cor0.__seg_id__), =($4, $cor0.gender))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), COALESCE(SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0))]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t16]) + EnumerableLimit(fetch=[10000]) + EnumerableHashJoin(condition=[AND(=($4, $13), =($11, $14), =($12, $15))], joinType=[left]) + EnumerableSort(sort0=[$11], dir0=[ASC]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[0], expr#17=[COALESCE($t15, $t16)], expr#18=[+($t14, $t17)], proj#0..11=[{exprs}], __seg_id__=[$t18]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($12)])], window#1=[window(rows between UNBOUNDED PRECEDING and $14 PRECEDING aggs [$SUM0($13)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], proj#0..10=[{exprs}], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t4, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t3)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t4)], proj#0..2=[{exprs}], avg_age=[$t10]) + EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($4)], agg#1=[COUNT($4)]) + EnumerableHashJoin(condition=[AND(=($2, $6), =($0, $3), <($5, $1))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2}]) + EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[COALESCE($t5, $t6)], expr#8=[+($t4, $t7)], proj#0..1=[{exprs}], __seg_id__=[$t8]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $4 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_take.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_take.json deleted file mode 100644 index cac543209cc..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_take.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], take=[TAKE($0, $1)])\n LogicalProject(firstname=[$1], $f1=[2])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableAggregate(group=[{}], take=[TAKE($0, $1)])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[2], firstname=[$t1], $f1=[$t17])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_take.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_take.yaml new file mode 100644 index 00000000000..0f98cbf8ab5 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_take.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], take=[TAKE($0, $1)]) + LogicalProject(firstname=[$1], $f1=[2]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableAggregate(group=[{}], take=[TAKE($0, $1)]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[2], firstname=[$t1], $f1=[$t17]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function.json deleted file mode 100644 index 385e760d366..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10])\n LogicalFilter(condition=[ILIKE($2, '%Holmes%':VARCHAR, '\\')])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%Holmes%':VARCHAR], expr#18=['\\'], expr#19=[ILIKE($t2, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function.yaml new file mode 100644 index 00000000000..41638cd1b16 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalFilter(condition=[ILIKE($2, '%Holmes%', '\')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%Holmes%'], expr#18=['\'], expr#19=[ILIKE($t2, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_no_pushdown.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart.yaml similarity index 100% rename from integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_no_pushdown.yaml rename to integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart.yaml diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count_no_pushdown.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart_count.yaml similarity index 75% rename from integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count_no_pushdown.yaml rename to integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart_count.yaml index 02f61d2177c..e60799af17d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count_no_pushdown.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart_count.yaml @@ -46,18 +46,16 @@ calcite: EnumerableUnion(all=[false]) EnumerableAggregate(group=[{0, 1}], actual_count=[$SUM0($2)]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t0):TIMESTAMP(0) NOT NULL], expr#6=[IS NOT NULL($t3)], expr#7=[IS NULL($t1)], expr#8=[null:NULL], expr#9=['OTHER'], expr#10=[CASE($t7, $t8, $t9)], expr#11=[CASE($t6, $t1, $t10)], @timestamp=[$t5], host=[$t11], count=[$t2]) - EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], @timestamp=[$t1], host=[$t0], $f2_0=[$t2]) - EnumerableAggregate(group=[{0, 1}], agg#0=[COUNT()]) - EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], host=[$t4], $f2=[$t18]) + EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[left]) + EnumerableCalc(expr#0..2=[{inputs}], @timestamp=[$t1], host=[$t0], $f2_0=[$t2]) + EnumerableAggregate(group=[{0, 1}], agg#0=[COUNT()]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], host=[$t4], $f2=[$t18]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) + EnumerableLimit(fetch=[10]) + EnumerableSort(sort0=[$1], dir0=[DESC]) + EnumerableAggregate(group=[{4}], grand_total=[COUNT()]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[IS NOT NULL($t4)], proj#0..15=[{exprs}], $condition=[$t16]) CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) - EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableLimit(fetch=[10]) - EnumerableSort(sort0=[$1], dir0=[DESC]) - EnumerableAggregate(group=[{4}], grand_total=[COUNT()]) - EnumerableCalc(expr#0..15=[{inputs}], expr#16=[IS NOT NULL($t4)], proj#0..15=[{exprs}], $condition=[$t16]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):TIMESTAMP(0) NOT NULL], expr#3=[0], @timestamp=[$t2], host=[$t1], count=[$t3]) EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) EnumerableAggregate(group=[{1}]) @@ -65,15 +63,13 @@ calcite: CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) EnumerableAggregate(group=[{0}]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t1)], expr#4=[IS NULL($t0)], expr#5=[null:NULL], expr#6=['OTHER'], expr#7=[CASE($t4, $t5, $t6)], expr#8=[CASE($t3, $t0, $t7)], $f0=[$t8]) - EnumerableMergeJoin(condition=[=($0, $1)], joinType=[left]) - EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..1=[{inputs}], host=[$t0]) - EnumerableAggregate(group=[{0, 1}]) - EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], host=[$t4], $f2=[$t18]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) - EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableLimit(fetch=[10]) - EnumerableSort(sort0=[$1], dir0=[DESC]) - EnumerableAggregate(group=[{4}], grand_total=[COUNT()]) - EnumerableCalc(expr#0..15=[{inputs}], expr#16=[IS NOT NULL($t4)], proj#0..15=[{exprs}], $condition=[$t16]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) + EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + EnumerableCalc(expr#0..1=[{inputs}], host=[$t0]) + EnumerableAggregate(group=[{0, 1}]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], host=[$t4], $f2=[$t18]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) + EnumerableLimit(fetch=[10]) + EnumerableSort(sort0=[$1], dir0=[DESC]) + EnumerableAggregate(group=[{4}], grand_total=[COUNT()]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[IS NOT NULL($t4)], proj#0..15=[{exprs}], $condition=[$t16]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_top_usenull_false.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_top_usenull_false.yaml new file mode 100644 index 00000000000..17a17081e61 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_top_usenull_false.yaml @@ -0,0 +1,17 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(gender=[$0], state=[$1], count=[$2]) + LogicalFilter(condition=[<=($3, 2)]) + LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 DESC)]) + LogicalAggregate(group=[{0, 1}], count=[COUNT()]) + LogicalProject(gender=[$4], state=[$7]) + LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[2], expr#5=[<=($t3, $t4)], proj#0..2=[{exprs}], $condition=[$t5]) + EnumerableWindow(window#0=[window(partition {0} order by [2 DESC] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableAggregate(group=[{4, 7}], count=[COUNT()]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NOT NULL($t4)], expr#18=[IS NOT NULL($t7)], expr#19=[AND($t17, $t18)], proj#0..16=[{exprs}], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_top_usenull_true.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_top_usenull_true.yaml new file mode 100644 index 00000000000..5fb27c851af --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_top_usenull_true.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(gender=[$0], state=[$1], count=[$2]) + LogicalFilter(condition=[<=($3, 2)]) + LogicalProject(gender=[$0], state=[$1], count=[$2], _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 DESC)]) + LogicalAggregate(group=[{0, 1}], count=[COUNT()]) + LogicalProject(gender=[$4], state=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[2], expr#5=[<=($t3, $t4)], proj#0..2=[{exprs}], $condition=[$t5]) + EnumerableWindow(window#0=[window(partition {0} order by [2 DESC] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableAggregate(group=[{4, 7}], count=[COUNT()]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_push.yaml index 74d2b3d4d49..0d929025da2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_push.yaml @@ -8,7 +8,7 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], expr#5=[>($t1, $t4)], expr#6=[CAST($t3):DOUBLE NOT NULL], expr#7=[/($t2, $t6)], expr#8=[null:NULL], expr#9=[CASE($t5, $t7, $t8)], ageTrend=[$t9]) - EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), $SUM0($0), COUNT($0)])]) + EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), $SUM0($0), COUNT($0)])], constants=[[1]]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NOT NULL($t8)], age=[$t8], $condition=[$t17]) EnumerableLimit(fetch=[5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_sort_push.yaml index ea522249b3a..2427a30e1a7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_sort_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_sort_push.yaml @@ -9,8 +9,8 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], expr#5=[>($t1, $t4)], expr#6=[CAST($t3):DOUBLE NOT NULL], expr#7=[/($t2, $t6)], expr#8=[null:NULL], expr#9=[CASE($t5, $t7, $t8)], ageTrend=[$t9]) - EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), $SUM0($0), COUNT($0)])]) + EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), $SUM0($0), COUNT($0)])], constants=[[1]]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NOT NULL($t8)], age=[$t8], $condition=[$t17]) EnumerableSort(sort0=[$8], dir0=[ASC]) EnumerableLimit(fetch=[5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/udf_geoip_in_agg_pushed.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/udf_geoip_in_agg_pushed.yaml new file mode 100644 index 00000000000..0dbea4e19e2 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/udf_geoip_in_agg_pushed.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], info.city=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(info.city=[ITEM(GEOIP('my-datasource':VARCHAR, $0), 'city')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], info.city=[$t0]) + EnumerableAggregate(group=[{0}], count()=[COUNT()]) + EnumerableCalc(expr#0..11=[{inputs}], expr#12=['my-datasource':VARCHAR], expr#13=[GEOIP($t12, $t0)], expr#14=['city'], expr#15=[ITEM($t13, $t14)], info.city=[$t15]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) diff --git a/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_timestamp.yaml new file mode 100644 index 00000000000..e57913c2f62 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_timestamp.yaml @@ -0,0 +1,16 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"agent\",\"process\"\ + ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ + ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]},\"sort\":[{\"@timestamp\":{\"order\":\"asc\",\"missing\":\"_first\"\ + }}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_timestamp_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_timestamp_can_match_shortcut.yaml new file mode 100644 index 00000000000..b0a8deed278 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_timestamp_can_match_shortcut.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"process.name:kernel\"\ + ,\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\"\ + :10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\"\ + :0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\"\ + :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ + ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ + aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\":\"\ + asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_timestamp_no_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_timestamp_no_can_match_shortcut.yaml new file mode 100644 index 00000000000..b0a8deed278 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_timestamp_no_can_match_shortcut.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"process.name:kernel\"\ + ,\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\"\ + :10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\"\ + :0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\"\ + :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ + ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ + aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\":\"\ + asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_with_after_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_with_after_timestamp.yaml new file mode 100644 index 00000000000..e57913c2f62 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_with_after_timestamp.yaml @@ -0,0 +1,16 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"agent\",\"process\"\ + ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ + ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]},\"sort\":[{\"@timestamp\":{\"order\":\"asc\",\"missing\":\"_first\"\ + }}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/cardinality_agg_high.yaml b/integ-test/src/test/resources/expectedOutput/ppl/cardinality_agg_high.yaml new file mode 100644 index 00000000000..ffe939e5a52 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/cardinality_agg_high.yaml @@ -0,0 +1,12 @@ +root: + name: ProjectOperator + description: + fields: "[dc(`agent.name`)]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"aggregations\":{\"dc(`agent.name`)\":{\"cardinality\"\ + :{\"field\":\"agent.name\"}}}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/cardinality_agg_high_2.yaml b/integ-test/src/test/resources/expectedOutput/ppl/cardinality_agg_high_2.yaml new file mode 100644 index 00000000000..0c147949642 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/cardinality_agg_high_2.yaml @@ -0,0 +1,12 @@ +root: + name: ProjectOperator + description: + fields: "[dc(`event.id`)]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"aggregations\":{\"dc(`event.id`)\":{\"cardinality\"\ + :{\"field\":\"event.id\"}}}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/cardinality_agg_low.yaml b/integ-test/src/test/resources/expectedOutput/ppl/cardinality_agg_low.yaml new file mode 100644 index 00000000000..f064201008e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/cardinality_agg_low.yaml @@ -0,0 +1,12 @@ +root: + name: ProjectOperator + description: + fields: "[dc(`cloud.region`)]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"aggregations\":{\"dc(`cloud.region`)\":{\"\ + cardinality\":{\"field\":\"cloud.region\"}}}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q1.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q1.yaml new file mode 100644 index 00000000000..969f430894d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q1.yaml @@ -0,0 +1,12 @@ +root: + name: ProjectOperator + description: + fields: "[count()]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"aggregations\":{\"count()\":{\"value_count\"\ + :{\"field\":\"_index\"}}}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q10.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q10.yaml new file mode 100644 index 00000000000..4ce4dfe5806 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q10.yaml @@ -0,0 +1,26 @@ +root: + name: ProjectOperator + description: + fields: "[sum(AdvEngineID), c, avg(ResolutionWidth), dc(UserID), RegionID]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 0 + sortList: + c: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\"\ + :{\"composite\":{\"size\":1000,\"sources\":[{\"RegionID\":{\"terms\"\ + :{\"field\":\"RegionID\",\"missing_bucket\":false,\"order\":\"asc\"\ + }}}]},\"aggregations\":{\"sum(AdvEngineID)\":{\"sum\":{\"field\":\"\ + AdvEngineID\"}},\"c\":{\"value_count\":{\"field\":\"_index\"}},\"avg(ResolutionWidth)\"\ + :{\"avg\":{\"field\":\"ResolutionWidth\"}},\"dc(UserID)\":{\"cardinality\"\ + :{\"field\":\"UserID\"}}}}}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q11.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q11.yaml new file mode 100644 index 00000000000..5d319c3819d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q11.yaml @@ -0,0 +1,26 @@ +root: + name: ProjectOperator + description: + fields: "[u, MobilePhoneModel]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 0 + sortList: + u: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"\ + source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0ABBNb2JpbGVQaG9uZU1vZGVsc3IAGmphdmEudXRpbC5BcnJheXMkQXJyYXlMaXN02aQ8vs2IBtICAAFbAAFhdAATW0xqYXZhL2xhbmcvT2JqZWN0O3hwdXIAE1tMamF2YS5sYW5nLlN0cmluZzut0lbn6R17RwIAAHhwAAAAAXEAfgANcQB+AA1+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAZTVFJJTkdzcgAvb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uTGl0ZXJhbEV4cHJlc3Npb25FQi3wjMeCJAIAAUwACWV4cHJWYWx1ZXQAKUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7eHBzcgAtb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuRXhwclN0cmluZ1ZhbHVlAEEyJXOJDhMCAAFMAAV2YWx1ZXEAfgALeHIAL29yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkFic3RyYWN0RXhwclZhbHVlyWu1dgYURIoCAAB4cHQAAHhzcgAzb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25OYW1lC6g4Tc72Z5cCAAFMAAxmdW5jdGlvbk5hbWVxAH4AC3hwdAACIT1xAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AIQAAAAZ1cQB+ACQAAAABc3EAfgAhAAAABnVxAH4AJAAAAAB2cgBJb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IucHJlZGljYXRlLkJpbmFyeVByZWRpY2F0ZU9wZXJhdG9ycwAAAAAAAAAAAAAAeHB0AD1vcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9udAAFYXBwbHl0ADgoTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3QASW9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL29wZXJhdG9yL3ByZWRpY2F0ZS9CaW5hcnlQcmVkaWNhdGVPcGVyYXRvcnN0ABpsYW1iZGEkbm90RXF1YWwkOTUwNDhmYzEkMXQAfShMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADF2cgAyb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25EU0wAAAAAAAAAAAAAAHhwcQB+ACxxAH4ALXEAfgAudAAyb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25EU0x0ACVsYW1iZGEkbnVsbE1pc3NpbmdIYW5kbGluZyRhNTAwNTI4MSQxdAC8KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AMXEAfgAzdAA+b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlVHJpRnVuY3Rpb25xAH4ALXQASihMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDspTGphdmEvbGFuZy9PYmplY3Q7cQB+ADR0ABZsYW1iZGEkaW1wbCRhMGZiMzRkNCQxdAD3KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3QAuChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AH3NyADlvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvblByb3BlcnRpZXPPPFljm6j5swIAA0wADWN1cnJlbnRab25lSWR0ABJMamF2YS90aW1lL1pvbmVJZDtMAApub3dJbnN0YW50dAATTGphdmEvdGltZS9JbnN0YW50O0wACXF1ZXJ5VHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhlY3V0b3IvUXVlcnlUeXBlO3hwc3IADWphdmEudGltZS5TZXKVXYS6GyJIsgwAAHhwdwIIAHhzcQB+AEF3DQIAAAAAaQmbFTUH9VB4fnIAJW9yZy5vcGVuc2VhcmNoLnNxbC5leGVjdXRvci5RdWVyeVR5cGUAAAAAAAAAABIAAHhxAH4AFHQAA1BQTH5xAH4AE3QAB0JPT0xFQU4=\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"},\"boost\":1.0}},\"\ + aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"\ + sources\":[{\"MobilePhoneModel\":{\"terms\":{\"field\":\"MobilePhoneModel\"\ + ,\"missing_bucket\":false,\"order\":\"asc\"}}}]},\"aggregations\":{\"\ + u\":{\"cardinality\":{\"field\":\"UserID\"}}}}}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q12.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q12.yaml new file mode 100644 index 00000000000..e1ea37a694b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q12.yaml @@ -0,0 +1,28 @@ +root: + name: ProjectOperator + description: + fields: "[u, MobilePhone, MobilePhoneModel]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 0 + sortList: + u: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"\ + source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0ABBNb2JpbGVQaG9uZU1vZGVsc3IAGmphdmEudXRpbC5BcnJheXMkQXJyYXlMaXN02aQ8vs2IBtICAAFbAAFhdAATW0xqYXZhL2xhbmcvT2JqZWN0O3hwdXIAE1tMamF2YS5sYW5nLlN0cmluZzut0lbn6R17RwIAAHhwAAAAAXEAfgANcQB+AA1+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAZTVFJJTkdzcgAvb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uTGl0ZXJhbEV4cHJlc3Npb25FQi3wjMeCJAIAAUwACWV4cHJWYWx1ZXQAKUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7eHBzcgAtb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuRXhwclN0cmluZ1ZhbHVlAEEyJXOJDhMCAAFMAAV2YWx1ZXEAfgALeHIAL29yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkFic3RyYWN0RXhwclZhbHVlyWu1dgYURIoCAAB4cHQAAHhzcgAzb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25OYW1lC6g4Tc72Z5cCAAFMAAxmdW5jdGlvbk5hbWVxAH4AC3hwdAACIT1xAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AIQAAAAZ1cQB+ACQAAAABc3EAfgAhAAAABnVxAH4AJAAAAAB2cgBJb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IucHJlZGljYXRlLkJpbmFyeVByZWRpY2F0ZU9wZXJhdG9ycwAAAAAAAAAAAAAAeHB0AD1vcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9udAAFYXBwbHl0ADgoTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3QASW9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL29wZXJhdG9yL3ByZWRpY2F0ZS9CaW5hcnlQcmVkaWNhdGVPcGVyYXRvcnN0ABpsYW1iZGEkbm90RXF1YWwkOTUwNDhmYzEkMXQAfShMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADF2cgAyb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25EU0wAAAAAAAAAAAAAAHhwcQB+ACxxAH4ALXEAfgAudAAyb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25EU0x0ACVsYW1iZGEkbnVsbE1pc3NpbmdIYW5kbGluZyRhNTAwNTI4MSQxdAC8KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AMXEAfgAzdAA+b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlVHJpRnVuY3Rpb25xAH4ALXQASihMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDspTGphdmEvbGFuZy9PYmplY3Q7cQB+ADR0ABZsYW1iZGEkaW1wbCRhMGZiMzRkNCQxdAD3KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3QAuChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AH3NyADlvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvblByb3BlcnRpZXPPPFljm6j5swIAA0wADWN1cnJlbnRab25lSWR0ABJMamF2YS90aW1lL1pvbmVJZDtMAApub3dJbnN0YW50dAATTGphdmEvdGltZS9JbnN0YW50O0wACXF1ZXJ5VHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhlY3V0b3IvUXVlcnlUeXBlO3hwc3IADWphdmEudGltZS5TZXKVXYS6GyJIsgwAAHhwdwIIAHhzcQB+AEF3DQIAAAAAaQmbFTtipuh4fnIAJW9yZy5vcGVuc2VhcmNoLnNxbC5leGVjdXRvci5RdWVyeVR5cGUAAAAAAAAAABIAAHhxAH4AFHQAA1BQTH5xAH4AE3QAB0JPT0xFQU4=\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"},\"boost\":1.0}},\"\ + aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"\ + sources\":[{\"MobilePhone\":{\"terms\":{\"field\":\"MobilePhone\",\"\ + missing_bucket\":false,\"order\":\"asc\"}}},{\"MobilePhoneModel\":{\"\ + terms\":{\"field\":\"MobilePhoneModel\",\"missing_bucket\":false,\"\ + order\":\"asc\"}}}]},\"aggregations\":{\"u\":{\"cardinality\":{\"field\"\ + :\"UserID\"}}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q13.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q13.yaml new file mode 100644 index 00000000000..82b6649c7e1 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q13.yaml @@ -0,0 +1,26 @@ +root: + name: ProjectOperator + description: + fields: "[c, SearchPhrase]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 0 + sortList: + c: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"\ + source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AAxTZWFyY2hQaHJhc2VzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABlNUUklOR3NyAC9vcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5MaXRlcmFsRXhwcmVzc2lvbkVCLfCMx4IkAgABTAAJZXhwclZhbHVldAApTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt4cHNyAC1vcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5FeHByU3RyaW5nVmFsdWUAQTIlc4kOEwIAAUwABXZhbHVlcQB+AAt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwdAAAeHNyADNvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbk5hbWULqDhNzvZnlwIAAUwADGZ1bmN0aW9uTmFtZXEAfgALeHB0AAIhPXEAfgAJc3IAIWphdmEubGFuZy5pbnZva2UuU2VyaWFsaXplZExhbWJkYW9h0JQsKTaFAgAKSQAOaW1wbE1ldGhvZEtpbmRbAAxjYXB0dXJlZEFyZ3NxAH4AD0wADmNhcHR1cmluZ0NsYXNzdAARTGphdmEvbGFuZy9DbGFzcztMABhmdW5jdGlvbmFsSW50ZXJmYWNlQ2xhc3NxAH4AC0wAHWZ1bmN0aW9uYWxJbnRlcmZhY2VNZXRob2ROYW1lcQB+AAtMACJmdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kU2lnbmF0dXJlcQB+AAtMAAlpbXBsQ2xhc3NxAH4AC0wADmltcGxNZXRob2ROYW1lcQB+AAtMABNpbXBsTWV0aG9kU2lnbmF0dXJlcQB+AAtMABZpbnN0YW50aWF0ZWRNZXRob2RUeXBlcQB+AAt4cAAAAAZ1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAABc3EAfgAhAAAABnVxAH4AJAAAAAFzcQB+ACEAAAAGdXEAfgAkAAAAAHZyAElvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5vcGVyYXRvci5wcmVkaWNhdGUuQmluYXJ5UHJlZGljYXRlT3BlcmF0b3JzAAAAAAAAAAAAAAB4cHQAPW9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb250AAVhcHBseXQAOChMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDspTGphdmEvbGFuZy9PYmplY3Q7dABJb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vb3BlcmF0b3IvcHJlZGljYXRlL0JpbmFyeVByZWRpY2F0ZU9wZXJhdG9yc3QAGmxhbWJkYSRub3RFcXVhbCQ5NTA0OGZjMSQxdAB9KExvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AMXZyADJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTAAAAAAAAAAAAAAAeHBxAH4ALHEAfgAtcQB+AC50ADJvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvbkRTTHQAJWxhbWJkYSRudWxsTWlzc2luZ0hhbmRsaW5nJGE1MDA1MjgxJDF0ALwoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAxcQB+ADN0AD5vcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbnEAfgAtdABKKExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDtxAH4ANHQAFmxhbWJkYSRpbXBsJGEwZmIzNGQ0JDF0APcoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7dAC4KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAfc3IAOW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uUHJvcGVydGllc888WWObqPmzAgADTAANY3VycmVudFpvbmVJZHQAEkxqYXZhL3RpbWUvWm9uZUlkO0wACm5vd0luc3RhbnR0ABNMamF2YS90aW1lL0luc3RhbnQ7TAAJcXVlcnlUeXBldAAnTG9yZy9vcGVuc2VhcmNoL3NxbC9leGVjdXRvci9RdWVyeVR5cGU7eHBzcgANamF2YS50aW1lLlNlcpVdhLobIkiyDAAAeHB3AggAeHNxAH4AQXcNAgAAAABpCZsWBTBK+Hh+cgAlb3JnLm9wZW5zZWFyY2guc3FsLmV4ZWN1dG9yLlF1ZXJ5VHlwZQAAAAAAAAAAEgAAeHEAfgAUdAADUFBMfnEAfgATdAAHQk9PTEVBTg==\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"},\"boost\":1.0}},\"\ + aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"\ + sources\":[{\"SearchPhrase\":{\"terms\":{\"field\":\"SearchPhrase\"\ + ,\"missing_bucket\":false,\"order\":\"asc\"}}}]},\"aggregations\":{\"\ + c\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q14.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q14.yaml new file mode 100644 index 00000000000..b85e2376de9 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q14.yaml @@ -0,0 +1,26 @@ +root: + name: ProjectOperator + description: + fields: "[u, SearchPhrase]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 0 + sortList: + u: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"\ + source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AAxTZWFyY2hQaHJhc2VzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABlNUUklOR3NyAC9vcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5MaXRlcmFsRXhwcmVzc2lvbkVCLfCMx4IkAgABTAAJZXhwclZhbHVldAApTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt4cHNyAC1vcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5FeHByU3RyaW5nVmFsdWUAQTIlc4kOEwIAAUwABXZhbHVlcQB+AAt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwdAAAeHNyADNvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbk5hbWULqDhNzvZnlwIAAUwADGZ1bmN0aW9uTmFtZXEAfgALeHB0AAIhPXEAfgAJc3IAIWphdmEubGFuZy5pbnZva2UuU2VyaWFsaXplZExhbWJkYW9h0JQsKTaFAgAKSQAOaW1wbE1ldGhvZEtpbmRbAAxjYXB0dXJlZEFyZ3NxAH4AD0wADmNhcHR1cmluZ0NsYXNzdAARTGphdmEvbGFuZy9DbGFzcztMABhmdW5jdGlvbmFsSW50ZXJmYWNlQ2xhc3NxAH4AC0wAHWZ1bmN0aW9uYWxJbnRlcmZhY2VNZXRob2ROYW1lcQB+AAtMACJmdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kU2lnbmF0dXJlcQB+AAtMAAlpbXBsQ2xhc3NxAH4AC0wADmltcGxNZXRob2ROYW1lcQB+AAtMABNpbXBsTWV0aG9kU2lnbmF0dXJlcQB+AAtMABZpbnN0YW50aWF0ZWRNZXRob2RUeXBlcQB+AAt4cAAAAAZ1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAABc3EAfgAhAAAABnVxAH4AJAAAAAFzcQB+ACEAAAAGdXEAfgAkAAAAAHZyAElvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5vcGVyYXRvci5wcmVkaWNhdGUuQmluYXJ5UHJlZGljYXRlT3BlcmF0b3JzAAAAAAAAAAAAAAB4cHQAPW9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb250AAVhcHBseXQAOChMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDspTGphdmEvbGFuZy9PYmplY3Q7dABJb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vb3BlcmF0b3IvcHJlZGljYXRlL0JpbmFyeVByZWRpY2F0ZU9wZXJhdG9yc3QAGmxhbWJkYSRub3RFcXVhbCQ5NTA0OGZjMSQxdAB9KExvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AMXZyADJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTAAAAAAAAAAAAAAAeHBxAH4ALHEAfgAtcQB+AC50ADJvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvbkRTTHQAJWxhbWJkYSRudWxsTWlzc2luZ0hhbmRsaW5nJGE1MDA1MjgxJDF0ALwoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAxcQB+ADN0AD5vcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbnEAfgAtdABKKExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDtxAH4ANHQAFmxhbWJkYSRpbXBsJGEwZmIzNGQ0JDF0APcoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7dAC4KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAfc3IAOW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uUHJvcGVydGllc888WWObqPmzAgADTAANY3VycmVudFpvbmVJZHQAEkxqYXZhL3RpbWUvWm9uZUlkO0wACm5vd0luc3RhbnR0ABNMamF2YS90aW1lL0luc3RhbnQ7TAAJcXVlcnlUeXBldAAnTG9yZy9vcGVuc2VhcmNoL3NxbC9leGVjdXRvci9RdWVyeVR5cGU7eHBzcgANamF2YS50aW1lLlNlcpVdhLobIkiyDAAAeHB3AggAeHNxAH4AQXcNAgAAAABpCZsWB1bv0Hh+cgAlb3JnLm9wZW5zZWFyY2guc3FsLmV4ZWN1dG9yLlF1ZXJ5VHlwZQAAAAAAAAAAEgAAeHEAfgAUdAADUFBMfnEAfgATdAAHQk9PTEVBTg==\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"},\"boost\":1.0}},\"\ + aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"\ + sources\":[{\"SearchPhrase\":{\"terms\":{\"field\":\"SearchPhrase\"\ + ,\"missing_bucket\":false,\"order\":\"asc\"}}}]},\"aggregations\":{\"\ + u\":{\"cardinality\":{\"field\":\"UserID\"}}}}}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q15.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q15.yaml new file mode 100644 index 00000000000..71888a91777 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q15.yaml @@ -0,0 +1,28 @@ +root: + name: ProjectOperator + description: + fields: "[c, SearchEngineID, SearchPhrase]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 0 + sortList: + c: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"\ + source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AAxTZWFyY2hQaHJhc2VzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABlNUUklOR3NyAC9vcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5MaXRlcmFsRXhwcmVzc2lvbkVCLfCMx4IkAgABTAAJZXhwclZhbHVldAApTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt4cHNyAC1vcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5FeHByU3RyaW5nVmFsdWUAQTIlc4kOEwIAAUwABXZhbHVlcQB+AAt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwdAAAeHNyADNvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbk5hbWULqDhNzvZnlwIAAUwADGZ1bmN0aW9uTmFtZXEAfgALeHB0AAIhPXEAfgAJc3IAIWphdmEubGFuZy5pbnZva2UuU2VyaWFsaXplZExhbWJkYW9h0JQsKTaFAgAKSQAOaW1wbE1ldGhvZEtpbmRbAAxjYXB0dXJlZEFyZ3NxAH4AD0wADmNhcHR1cmluZ0NsYXNzdAARTGphdmEvbGFuZy9DbGFzcztMABhmdW5jdGlvbmFsSW50ZXJmYWNlQ2xhc3NxAH4AC0wAHWZ1bmN0aW9uYWxJbnRlcmZhY2VNZXRob2ROYW1lcQB+AAtMACJmdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kU2lnbmF0dXJlcQB+AAtMAAlpbXBsQ2xhc3NxAH4AC0wADmltcGxNZXRob2ROYW1lcQB+AAtMABNpbXBsTWV0aG9kU2lnbmF0dXJlcQB+AAtMABZpbnN0YW50aWF0ZWRNZXRob2RUeXBlcQB+AAt4cAAAAAZ1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAABc3EAfgAhAAAABnVxAH4AJAAAAAFzcQB+ACEAAAAGdXEAfgAkAAAAAHZyAElvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5vcGVyYXRvci5wcmVkaWNhdGUuQmluYXJ5UHJlZGljYXRlT3BlcmF0b3JzAAAAAAAAAAAAAAB4cHQAPW9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb250AAVhcHBseXQAOChMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDspTGphdmEvbGFuZy9PYmplY3Q7dABJb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vb3BlcmF0b3IvcHJlZGljYXRlL0JpbmFyeVByZWRpY2F0ZU9wZXJhdG9yc3QAGmxhbWJkYSRub3RFcXVhbCQ5NTA0OGZjMSQxdAB9KExvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AMXZyADJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTAAAAAAAAAAAAAAAeHBxAH4ALHEAfgAtcQB+AC50ADJvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvbkRTTHQAJWxhbWJkYSRudWxsTWlzc2luZ0hhbmRsaW5nJGE1MDA1MjgxJDF0ALwoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAxcQB+ADN0AD5vcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbnEAfgAtdABKKExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDtxAH4ANHQAFmxhbWJkYSRpbXBsJGEwZmIzNGQ0JDF0APcoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7dAC4KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAfc3IAOW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uUHJvcGVydGllc888WWObqPmzAgADTAANY3VycmVudFpvbmVJZHQAEkxqYXZhL3RpbWUvWm9uZUlkO0wACm5vd0luc3RhbnR0ABNMamF2YS90aW1lL0luc3RhbnQ7TAAJcXVlcnlUeXBldAAnTG9yZy9vcGVuc2VhcmNoL3NxbC9leGVjdXRvci9RdWVyeVR5cGU7eHBzcgANamF2YS50aW1lLlNlcpVdhLobIkiyDAAAeHB3AggAeHNxAH4AQXcNAgAAAABpCZsWCo/nOHh+cgAlb3JnLm9wZW5zZWFyY2guc3FsLmV4ZWN1dG9yLlF1ZXJ5VHlwZQAAAAAAAAAAEgAAeHEAfgAUdAADUFBMfnEAfgATdAAHQk9PTEVBTg==\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"},\"boost\":1.0}},\"\ + aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"\ + sources\":[{\"SearchEngineID\":{\"terms\":{\"field\":\"SearchEngineID\"\ + ,\"missing_bucket\":false,\"order\":\"asc\"}}},{\"SearchPhrase\":{\"\ + terms\":{\"field\":\"SearchPhrase\",\"missing_bucket\":false,\"order\"\ + :\"asc\"}}}]},\"aggregations\":{\"c\":{\"value_count\":{\"field\":\"\ + _index\"}}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q16.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q16.yaml new file mode 100644 index 00000000000..c1d24ad4c2b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q16.yaml @@ -0,0 +1,24 @@ +root: + name: ProjectOperator + description: + fields: "[count(), UserID]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 0 + sortList: + count(): + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\"\ + :{\"composite\":{\"size\":1000,\"sources\":[{\"UserID\":{\"terms\":{\"\ + field\":\"UserID\",\"missing_bucket\":false,\"order\":\"asc\"}}}]},\"\ + aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}},\ + \ needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q17.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q17.yaml new file mode 100644 index 00000000000..c376f075b21 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q17.yaml @@ -0,0 +1,25 @@ +root: + name: ProjectOperator + description: + fields: "[count(), UserID, SearchPhrase]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 0 + sortList: + count(): + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\"\ + :{\"composite\":{\"size\":1000,\"sources\":[{\"UserID\":{\"terms\":{\"\ + field\":\"UserID\",\"missing_bucket\":false,\"order\":\"asc\"}}},{\"\ + SearchPhrase\":{\"terms\":{\"field\":\"SearchPhrase\",\"missing_bucket\"\ + :false,\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\"\ + :{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q18.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q18.yaml new file mode 100644 index 00000000000..4a1e9941c20 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q18.yaml @@ -0,0 +1,21 @@ +root: + name: ProjectOperator + description: + fields: "[count(), UserID, SearchPhrase]" + children: + - name: LimitOperator + description: + limit: 10 + offset: 0 + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\"\ + :{\"composite\":{\"size\":1000,\"sources\":[{\"UserID\":{\"terms\":{\"\ + field\":\"UserID\",\"missing_bucket\":false,\"order\":\"asc\"}}},{\"\ + SearchPhrase\":{\"terms\":{\"field\":\"SearchPhrase\",\"missing_bucket\"\ + :false,\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\"\ + :{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q19.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q19.yaml new file mode 100644 index 00000000000..55ace7f34c2 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q19.yaml @@ -0,0 +1,31 @@ +root: + name: ProjectOperator + description: + fields: "[count(), UserID, m, SearchPhrase]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 0 + sortList: + count(): + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: AggregationOperator + description: + aggregators: "[count()]" + groupBy: "[UserID, m, SearchPhrase]" + children: + - name: OpenSearchEvalOperator + description: + expressions: + m: "extract(\"minute\", EventTime)" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"\ + from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true,\ + \ searchDone=false, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q2.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q2.yaml new file mode 100644 index 00000000000..b02293e8292 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q2.yaml @@ -0,0 +1,15 @@ +root: + name: ProjectOperator + description: + fields: "[count()]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\"\ + :\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AAtBZHZFbmdpbmVJRHNyABpqYXZhLnV0aWwuQXJyYXlzJEFycmF5TGlzdNmkPL7NiAbSAgABWwABYXQAE1tMamF2YS9sYW5nL09iamVjdDt4cHVyABNbTGphdmEubGFuZy5TdHJpbmc7rdJW5+kde0cCAAB4cAAAAAFxAH4ADXEAfgANfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAFU0hPUlRzcgAvb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uTGl0ZXJhbEV4cHJlc3Npb25FQi3wjMeCJAIAAUwACWV4cHJWYWx1ZXQAKUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7eHBzcgAub3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuRXhwckludGVnZXJWYWx1ZaZsB5mJt25DAgAAeHIANW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkFic3RyYWN0RXhwck51bWJlclZhbHVlNPRi6sWnMjsCAAFMAAV2YWx1ZXQAEkxqYXZhL2xhbmcvTnVtYmVyO3hyAC9vcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJWYWx1ZclrtXYGFESKAgAAeHBzcgARamF2YS5sYW5nLkludGVnZXIS4qCk94GHOAIAAUkABXZhbHVleHIAEGphdmEubGFuZy5OdW1iZXKGrJUdC5TgiwIAAHhwAAAAAHhzcgAzb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25OYW1lC6g4Tc72Z5cCAAFMAAxmdW5jdGlvbk5hbWVxAH4AC3hwdAACIT1xAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBJb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IucHJlZGljYXRlLkJpbmFyeVByZWRpY2F0ZU9wZXJhdG9ycwAAAAAAAAAAAAAAeHB0AD1vcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9udAAFYXBwbHl0ADgoTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3QASW9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL29wZXJhdG9yL3ByZWRpY2F0ZS9CaW5hcnlQcmVkaWNhdGVPcGVyYXRvcnN0ABpsYW1iZGEkbm90RXF1YWwkOTUwNDhmYzEkMXQAfShMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADV2cgAyb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25EU0wAAAAAAAAAAAAAAHhwcQB+ADBxAH4AMXEAfgAydAAyb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25EU0x0ACVsYW1iZGEkbnVsbE1pc3NpbmdIYW5kbGluZyRhNTAwNTI4MSQxdAC8KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4ANXEAfgA3dAA+b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlVHJpRnVuY3Rpb25xAH4AMXQASihMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDspTGphdmEvbGFuZy9PYmplY3Q7cQB+ADh0ABZsYW1iZGEkaW1wbCRhMGZiMzRkNCQxdAD3KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3QAuChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AI3NyADlvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvblByb3BlcnRpZXPPPFljm6j5swIAA0wADWN1cnJlbnRab25lSWR0ABJMamF2YS90aW1lL1pvbmVJZDtMAApub3dJbnN0YW50dAATTGphdmEvdGltZS9JbnN0YW50O0wACXF1ZXJ5VHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhlY3V0b3IvUXVlcnlUeXBlO3hwc3IADWphdmEudGltZS5TZXKVXYS6GyJIsgwAAHhwdwIIAHhzcQB+AEV3DQIAAAAAaQmbFRxMvFh4fnIAJW9yZy5vcGVuc2VhcmNoLnNxbC5leGVjdXRvci5RdWVyeVR5cGUAAAAAAAAAABIAAHhxAH4AFHQAA1BQTH5xAH4AE3QAB0JPT0xFQU4=\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"},\"boost\":1.0}},\"aggregations\"\ + :{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q20.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q20.yaml new file mode 100644 index 00000000000..243190d4116 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q20.yaml @@ -0,0 +1,13 @@ +root: + name: ProjectOperator + description: + fields: "[UserID]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\":0,\"\ + size\":10000,\"timeout\":\"1m\",\"query\":{\"term\":{\"UserID\":{\"value\"\ + :435090932899640449,\"boost\":1.0}}},\"_source\":{\"includes\":[\"UserID\"\ + ],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q21.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q21.yaml new file mode 100644 index 00000000000..6e21d7309ec --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q21.yaml @@ -0,0 +1,14 @@ +root: + name: ProjectOperator + description: + fields: "[count()]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"query\":{\"wildcard\":{\"URL\":{\"wildcard\"\ + :\"*google*\",\"case_insensitive\":true,\"boost\":1.0}}},\"aggregations\"\ + :{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q22.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q22.yaml new file mode 100644 index 00000000000..b1e81ba591b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q22.yaml @@ -0,0 +1,29 @@ +root: + name: ProjectOperator + description: + fields: "[c, SearchPhrase]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 0 + sortList: + c: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"\ + wildcard\":{\"URL\":{\"wildcard\":\"*google*\",\"case_insensitive\"\ + :true,\"boost\":1.0}}},{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\ + \":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AAxTZWFyY2hQaHJhc2VzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABlNUUklOR3NyAC9vcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5MaXRlcmFsRXhwcmVzc2lvbkVCLfCMx4IkAgABTAAJZXhwclZhbHVldAApTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt4cHNyAC1vcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5FeHByU3RyaW5nVmFsdWUAQTIlc4kOEwIAAUwABXZhbHVlcQB+AAt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwdAAAeHNyADNvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbk5hbWULqDhNzvZnlwIAAUwADGZ1bmN0aW9uTmFtZXEAfgALeHB0AAIhPXEAfgAJc3IAIWphdmEubGFuZy5pbnZva2UuU2VyaWFsaXplZExhbWJkYW9h0JQsKTaFAgAKSQAOaW1wbE1ldGhvZEtpbmRbAAxjYXB0dXJlZEFyZ3NxAH4AD0wADmNhcHR1cmluZ0NsYXNzdAARTGphdmEvbGFuZy9DbGFzcztMABhmdW5jdGlvbmFsSW50ZXJmYWNlQ2xhc3NxAH4AC0wAHWZ1bmN0aW9uYWxJbnRlcmZhY2VNZXRob2ROYW1lcQB+AAtMACJmdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kU2lnbmF0dXJlcQB+AAtMAAlpbXBsQ2xhc3NxAH4AC0wADmltcGxNZXRob2ROYW1lcQB+AAtMABNpbXBsTWV0aG9kU2lnbmF0dXJlcQB+AAtMABZpbnN0YW50aWF0ZWRNZXRob2RUeXBlcQB+AAt4cAAAAAZ1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAABc3EAfgAhAAAABnVxAH4AJAAAAAFzcQB+ACEAAAAGdXEAfgAkAAAAAHZyAElvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5vcGVyYXRvci5wcmVkaWNhdGUuQmluYXJ5UHJlZGljYXRlT3BlcmF0b3JzAAAAAAAAAAAAAAB4cHQAPW9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb250AAVhcHBseXQAOChMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDspTGphdmEvbGFuZy9PYmplY3Q7dABJb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vb3BlcmF0b3IvcHJlZGljYXRlL0JpbmFyeVByZWRpY2F0ZU9wZXJhdG9yc3QAGmxhbWJkYSRub3RFcXVhbCQ5NTA0OGZjMSQxdAB9KExvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AMXZyADJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTAAAAAAAAAAAAAAAeHBxAH4ALHEAfgAtcQB+AC50ADJvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvbkRTTHQAJWxhbWJkYSRudWxsTWlzc2luZ0hhbmRsaW5nJGE1MDA1MjgxJDF0ALwoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAxcQB+ADN0AD5vcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbnEAfgAtdABKKExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDtxAH4ANHQAFmxhbWJkYSRpbXBsJGEwZmIzNGQ0JDF0APcoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7dAC4KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAfc3IAOW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uUHJvcGVydGllc888WWObqPmzAgADTAANY3VycmVudFpvbmVJZHQAEkxqYXZhL3RpbWUvWm9uZUlkO0wACm5vd0luc3RhbnR0ABNMamF2YS90aW1lL0luc3RhbnQ7TAAJcXVlcnlUeXBldAAnTG9yZy9vcGVuc2VhcmNoL3NxbC9leGVjdXRvci9RdWVyeVR5cGU7eHBzcgANamF2YS50aW1lLlNlcpVdhLobIkiyDAAAeHB3AggAeHNxAH4AQXcNAgAAAABpCZsWGnrS6Hh+cgAlb3JnLm9wZW5zZWFyY2guc3FsLmV4ZWN1dG9yLlF1ZXJ5VHlwZQAAAAAAAAAAEgAAeHEAfgAUdAADUFBMfnEAfgATdAAHQk9PTEVBTg==\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"},\"boost\":1.0}}],\"\ + adjust_pure_negative\":true,\"boost\":1.0}},\"aggregations\":{\"composite_buckets\"\ + :{\"composite\":{\"size\":1000,\"sources\":[{\"SearchPhrase\":{\"terms\"\ + :{\"field\":\"SearchPhrase\",\"missing_bucket\":false,\"order\":\"asc\"\ + }}}]},\"aggregations\":{\"c\":{\"value_count\":{\"field\":\"_index\"\ + }}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q23.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q23.yaml new file mode 100644 index 00000000000..6c68c200dce --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q23.yaml @@ -0,0 +1,33 @@ +root: + name: ProjectOperator + description: + fields: "[c, dc(UserID), SearchPhrase]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 0 + sortList: + c: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"\ + bool\":{\"filter\":[{\"wildcard\":{\"Title\":{\"wildcard\":\"*Google*\"\ + ,\"case_insensitive\":true,\"boost\":1.0}}},{\"bool\":{\"must_not\"\ + :[{\"wildcard\":{\"URL\":{\"wildcard\":\"*.google.*\",\"case_insensitive\"\ + :true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}}],\"\ + adjust_pure_negative\":true,\"boost\":1.0}},{\"script\":{\"script\"\ + :{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AAxTZWFyY2hQaHJhc2VzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABlNUUklOR3NyAC9vcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5MaXRlcmFsRXhwcmVzc2lvbkVCLfCMx4IkAgABTAAJZXhwclZhbHVldAApTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt4cHNyAC1vcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5FeHByU3RyaW5nVmFsdWUAQTIlc4kOEwIAAUwABXZhbHVlcQB+AAt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwdAAAeHNyADNvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbk5hbWULqDhNzvZnlwIAAUwADGZ1bmN0aW9uTmFtZXEAfgALeHB0AAIhPXEAfgAJc3IAIWphdmEubGFuZy5pbnZva2UuU2VyaWFsaXplZExhbWJkYW9h0JQsKTaFAgAKSQAOaW1wbE1ldGhvZEtpbmRbAAxjYXB0dXJlZEFyZ3NxAH4AD0wADmNhcHR1cmluZ0NsYXNzdAARTGphdmEvbGFuZy9DbGFzcztMABhmdW5jdGlvbmFsSW50ZXJmYWNlQ2xhc3NxAH4AC0wAHWZ1bmN0aW9uYWxJbnRlcmZhY2VNZXRob2ROYW1lcQB+AAtMACJmdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kU2lnbmF0dXJlcQB+AAtMAAlpbXBsQ2xhc3NxAH4AC0wADmltcGxNZXRob2ROYW1lcQB+AAtMABNpbXBsTWV0aG9kU2lnbmF0dXJlcQB+AAtMABZpbnN0YW50aWF0ZWRNZXRob2RUeXBlcQB+AAt4cAAAAAZ1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAABc3EAfgAhAAAABnVxAH4AJAAAAAFzcQB+ACEAAAAGdXEAfgAkAAAAAHZyAElvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5vcGVyYXRvci5wcmVkaWNhdGUuQmluYXJ5UHJlZGljYXRlT3BlcmF0b3JzAAAAAAAAAAAAAAB4cHQAPW9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb250AAVhcHBseXQAOChMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDspTGphdmEvbGFuZy9PYmplY3Q7dABJb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vb3BlcmF0b3IvcHJlZGljYXRlL0JpbmFyeVByZWRpY2F0ZU9wZXJhdG9yc3QAGmxhbWJkYSRub3RFcXVhbCQ5NTA0OGZjMSQxdAB9KExvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AMXZyADJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTAAAAAAAAAAAAAAAeHBxAH4ALHEAfgAtcQB+AC50ADJvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvbkRTTHQAJWxhbWJkYSRudWxsTWlzc2luZ0hhbmRsaW5nJGE1MDA1MjgxJDF0ALwoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAxcQB+ADN0AD5vcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbnEAfgAtdABKKExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDtxAH4ANHQAFmxhbWJkYSRpbXBsJGEwZmIzNGQ0JDF0APcoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7dAC4KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAfc3IAOW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uUHJvcGVydGllc888WWObqPmzAgADTAANY3VycmVudFpvbmVJZHQAEkxqYXZhL3RpbWUvWm9uZUlkO0wACm5vd0luc3RhbnR0ABNMamF2YS90aW1lL0luc3RhbnQ7TAAJcXVlcnlUeXBldAAnTG9yZy9vcGVuc2VhcmNoL3NxbC9leGVjdXRvci9RdWVyeVR5cGU7eHBzcgANamF2YS50aW1lLlNlcpVdhLobIkiyDAAAeHB3AggAeHNxAH4AQXcNAgAAAABpCZsWHHwlEHh+cgAlb3JnLm9wZW5zZWFyY2guc3FsLmV4ZWN1dG9yLlF1ZXJ5VHlwZQAAAAAAAAAAEgAAeHEAfgAUdAADUFBMfnEAfgATdAAHQk9PTEVBTg==\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"},\"boost\":1.0}}],\"\ + adjust_pure_negative\":true,\"boost\":1.0}},\"aggregations\":{\"composite_buckets\"\ + :{\"composite\":{\"size\":1000,\"sources\":[{\"SearchPhrase\":{\"terms\"\ + :{\"field\":\"SearchPhrase\",\"missing_bucket\":false,\"order\":\"asc\"\ + }}}]},\"aggregations\":{\"c\":{\"value_count\":{\"field\":\"_index\"\ + }},\"dc(UserID)\":{\"cardinality\":{\"field\":\"UserID\"}}}}}}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q24.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q24.yaml new file mode 100644 index 00000000000..da2a8eae916 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q24.yaml @@ -0,0 +1,52 @@ +root: + name: ProjectOperator + description: + fields: "[EventDate, URLRegionID, HasGCLID, Income, Interests, Robotness, BrowserLanguage,\ + \ CounterClass, BrowserCountry, OriginalURL, ClientTimeZone, RefererHash, TraficSourceID,\ + \ HitColor, RefererRegionID, URLCategoryID, LocalEventTime, EventTime, UTMTerm,\ + \ AdvEngineID, UserAgentMinor, UserAgentMajor, RemoteIP, Sex, JavaEnable, URLHash,\ + \ URL, ParamOrderID, OpenstatSourceID, HTTPError, SilverlightVersion3, MobilePhoneModel,\ + \ SilverlightVersion4, SilverlightVersion1, SilverlightVersion2, IsDownload,\ + \ IsParameter, CLID, FlashMajor, FlashMinor, UTMMedium, WatchID, DontCountHits,\ + \ CookieEnable, HID, SocialAction, WindowName, ConnectTiming, PageCharset, IsLink,\ + \ IsArtifical, JavascriptEnable, ClientEventTime, DNSTiming, CodeVersion, ResponseEndTiming,\ + \ FUniqID, WindowClientHeight, OpenstatServiceName, UTMContent, HistoryLength,\ + \ IsOldCounter, MobilePhone, SearchPhrase, FlashMinor2, SearchEngineID, IsEvent,\ + \ UTMSource, RegionID, OpenstatAdID, UTMCampaign, GoodEvent, IsRefresh, ParamCurrency,\ + \ Params, ResolutionHeight, ClientIP, FromTag, ParamCurrencyID, ResponseStartTiming,\ + \ ResolutionWidth, SendTiming, RefererCategoryID, OpenstatCampaignID, UserID,\ + \ WithHash, UserAgent, ParamPrice, ResolutionDepth, IsMobile, Age, SocialSourceNetworkID,\ + \ OpenerName, OS, IsNotBounce, Referer, NetMinor, Title, NetMajor, IPNetworkID,\ + \ FetchTiming, SocialNetwork, SocialSourcePage, CounterID, WindowClientWidth]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"wildcard\":{\"URL\":{\"wildcard\"\ + :\"*google*\",\"case_insensitive\":true,\"boost\":1.0}}},\"_source\":{\"\ + includes\":[\"EventDate\",\"URLRegionID\",\"HasGCLID\",\"Income\",\"Interests\"\ + ,\"Robotness\",\"BrowserLanguage\",\"CounterClass\",\"BrowserCountry\",\"\ + OriginalURL\",\"ClientTimeZone\",\"RefererHash\",\"TraficSourceID\",\"HitColor\"\ + ,\"RefererRegionID\",\"URLCategoryID\",\"LocalEventTime\",\"EventTime\"\ + ,\"UTMTerm\",\"AdvEngineID\",\"UserAgentMinor\",\"UserAgentMajor\",\"RemoteIP\"\ + ,\"Sex\",\"JavaEnable\",\"URLHash\",\"URL\",\"ParamOrderID\",\"OpenstatSourceID\"\ + ,\"HTTPError\",\"SilverlightVersion3\",\"MobilePhoneModel\",\"SilverlightVersion4\"\ + ,\"SilverlightVersion1\",\"SilverlightVersion2\",\"IsDownload\",\"IsParameter\"\ + ,\"CLID\",\"FlashMajor\",\"FlashMinor\",\"UTMMedium\",\"WatchID\",\"DontCountHits\"\ + ,\"CookieEnable\",\"HID\",\"SocialAction\",\"WindowName\",\"ConnectTiming\"\ + ,\"PageCharset\",\"IsLink\",\"IsArtifical\",\"JavascriptEnable\",\"ClientEventTime\"\ + ,\"DNSTiming\",\"CodeVersion\",\"ResponseEndTiming\",\"FUniqID\",\"WindowClientHeight\"\ + ,\"OpenstatServiceName\",\"UTMContent\",\"HistoryLength\",\"IsOldCounter\"\ + ,\"MobilePhone\",\"SearchPhrase\",\"FlashMinor2\",\"SearchEngineID\",\"\ + IsEvent\",\"UTMSource\",\"RegionID\",\"OpenstatAdID\",\"UTMCampaign\",\"\ + GoodEvent\",\"IsRefresh\",\"ParamCurrency\",\"Params\",\"ResolutionHeight\"\ + ,\"ClientIP\",\"FromTag\",\"ParamCurrencyID\",\"ResponseStartTiming\",\"\ + ResolutionWidth\",\"SendTiming\",\"RefererCategoryID\",\"OpenstatCampaignID\"\ + ,\"UserID\",\"WithHash\",\"UserAgent\",\"ParamPrice\",\"ResolutionDepth\"\ + ,\"IsMobile\",\"Age\",\"SocialSourceNetworkID\",\"OpenerName\",\"OS\",\"\ + IsNotBounce\",\"Referer\",\"NetMinor\",\"Title\",\"NetMajor\",\"IPNetworkID\"\ + ,\"FetchTiming\",\"SocialNetwork\",\"SocialSourcePage\",\"CounterID\",\"\ + WindowClientWidth\"],\"excludes\":[]},\"sort\":[{\"EventTime\":{\"order\"\ + :\"asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q25.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q25.yaml new file mode 100644 index 00000000000..07c3979ea39 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q25.yaml @@ -0,0 +1,26 @@ +root: + name: ProjectOperator + description: + fields: "[SearchPhrase]" + children: + - name: LimitOperator + description: + limit: 10 + offset: 0 + children: + - name: ProjectOperator + description: + fields: "[SearchPhrase]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"\ + from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"script\"\ + :{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AAxTZWFyY2hQaHJhc2VzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABlNUUklOR3NyAC9vcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5MaXRlcmFsRXhwcmVzc2lvbkVCLfCMx4IkAgABTAAJZXhwclZhbHVldAApTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt4cHNyAC1vcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5FeHByU3RyaW5nVmFsdWUAQTIlc4kOEwIAAUwABXZhbHVlcQB+AAt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwdAAAeHNyADNvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbk5hbWULqDhNzvZnlwIAAUwADGZ1bmN0aW9uTmFtZXEAfgALeHB0AAIhPXEAfgAJc3IAIWphdmEubGFuZy5pbnZva2UuU2VyaWFsaXplZExhbWJkYW9h0JQsKTaFAgAKSQAOaW1wbE1ldGhvZEtpbmRbAAxjYXB0dXJlZEFyZ3NxAH4AD0wADmNhcHR1cmluZ0NsYXNzdAARTGphdmEvbGFuZy9DbGFzcztMABhmdW5jdGlvbmFsSW50ZXJmYWNlQ2xhc3NxAH4AC0wAHWZ1bmN0aW9uYWxJbnRlcmZhY2VNZXRob2ROYW1lcQB+AAtMACJmdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kU2lnbmF0dXJlcQB+AAtMAAlpbXBsQ2xhc3NxAH4AC0wADmltcGxNZXRob2ROYW1lcQB+AAtMABNpbXBsTWV0aG9kU2lnbmF0dXJlcQB+AAtMABZpbnN0YW50aWF0ZWRNZXRob2RUeXBlcQB+AAt4cAAAAAZ1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAABc3EAfgAhAAAABnVxAH4AJAAAAAFzcQB+ACEAAAAGdXEAfgAkAAAAAHZyAElvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5vcGVyYXRvci5wcmVkaWNhdGUuQmluYXJ5UHJlZGljYXRlT3BlcmF0b3JzAAAAAAAAAAAAAAB4cHQAPW9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb250AAVhcHBseXQAOChMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDspTGphdmEvbGFuZy9PYmplY3Q7dABJb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vb3BlcmF0b3IvcHJlZGljYXRlL0JpbmFyeVByZWRpY2F0ZU9wZXJhdG9yc3QAGmxhbWJkYSRub3RFcXVhbCQ5NTA0OGZjMSQxdAB9KExvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AMXZyADJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTAAAAAAAAAAAAAAAeHBxAH4ALHEAfgAtcQB+AC50ADJvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvbkRTTHQAJWxhbWJkYSRudWxsTWlzc2luZ0hhbmRsaW5nJGE1MDA1MjgxJDF0ALwoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAxcQB+ADN0AD5vcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbnEAfgAtdABKKExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDtxAH4ANHQAFmxhbWJkYSRpbXBsJGEwZmIzNGQ0JDF0APcoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7dAC4KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAfc3IAOW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uUHJvcGVydGllc888WWObqPmzAgADTAANY3VycmVudFpvbmVJZHQAEkxqYXZhL3RpbWUvWm9uZUlkO0wACm5vd0luc3RhbnR0ABNMamF2YS90aW1lL0luc3RhbnQ7TAAJcXVlcnlUeXBldAAnTG9yZy9vcGVuc2VhcmNoL3NxbC9leGVjdXRvci9RdWVyeVR5cGU7eHBzcgANamF2YS50aW1lLlNlcpVdhLobIkiyDAAAeHB3AggAeHNxAH4AQXcNAgAAAABpCZsWIFYPmHh+cgAlb3JnLm9wZW5zZWFyY2guc3FsLmV4ZWN1dG9yLlF1ZXJ5VHlwZQAAAAAAAAAAEgAAeHEAfgAUdAADUFBMfnEAfgATdAAHQk9PTEVBTg==\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"},\"boost\":1.0}},\"\ + _source\":{\"includes\":[\"SearchPhrase\"],\"excludes\":[]},\"sort\"\ + :[{\"EventTime\":{\"order\":\"asc\",\"missing\":\"_first\"}}]},\ + \ needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q26.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q26.yaml new file mode 100644 index 00000000000..b7437d23381 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q26.yaml @@ -0,0 +1,29 @@ +root: + name: ProjectOperator + description: + fields: "[SearchPhrase]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 0 + sortList: + SearchPhrase: + sortOrder: ASC + nullOrder: NULL_FIRST + children: + - name: ProjectOperator + description: + fields: "[SearchPhrase]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"\ + from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"script\"\ + :{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AAxTZWFyY2hQaHJhc2VzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABlNUUklOR3NyAC9vcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5MaXRlcmFsRXhwcmVzc2lvbkVCLfCMx4IkAgABTAAJZXhwclZhbHVldAApTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt4cHNyAC1vcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5FeHByU3RyaW5nVmFsdWUAQTIlc4kOEwIAAUwABXZhbHVlcQB+AAt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwdAAAeHNyADNvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbk5hbWULqDhNzvZnlwIAAUwADGZ1bmN0aW9uTmFtZXEAfgALeHB0AAIhPXEAfgAJc3IAIWphdmEubGFuZy5pbnZva2UuU2VyaWFsaXplZExhbWJkYW9h0JQsKTaFAgAKSQAOaW1wbE1ldGhvZEtpbmRbAAxjYXB0dXJlZEFyZ3NxAH4AD0wADmNhcHR1cmluZ0NsYXNzdAARTGphdmEvbGFuZy9DbGFzcztMABhmdW5jdGlvbmFsSW50ZXJmYWNlQ2xhc3NxAH4AC0wAHWZ1bmN0aW9uYWxJbnRlcmZhY2VNZXRob2ROYW1lcQB+AAtMACJmdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kU2lnbmF0dXJlcQB+AAtMAAlpbXBsQ2xhc3NxAH4AC0wADmltcGxNZXRob2ROYW1lcQB+AAtMABNpbXBsTWV0aG9kU2lnbmF0dXJlcQB+AAtMABZpbnN0YW50aWF0ZWRNZXRob2RUeXBlcQB+AAt4cAAAAAZ1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAABc3EAfgAhAAAABnVxAH4AJAAAAAFzcQB+ACEAAAAGdXEAfgAkAAAAAHZyAElvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5vcGVyYXRvci5wcmVkaWNhdGUuQmluYXJ5UHJlZGljYXRlT3BlcmF0b3JzAAAAAAAAAAAAAAB4cHQAPW9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb250AAVhcHBseXQAOChMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDspTGphdmEvbGFuZy9PYmplY3Q7dABJb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vb3BlcmF0b3IvcHJlZGljYXRlL0JpbmFyeVByZWRpY2F0ZU9wZXJhdG9yc3QAGmxhbWJkYSRub3RFcXVhbCQ5NTA0OGZjMSQxdAB9KExvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AMXZyADJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTAAAAAAAAAAAAAAAeHBxAH4ALHEAfgAtcQB+AC50ADJvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvbkRTTHQAJWxhbWJkYSRudWxsTWlzc2luZ0hhbmRsaW5nJGE1MDA1MjgxJDF0ALwoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAxcQB+ADN0AD5vcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbnEAfgAtdABKKExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDtxAH4ANHQAFmxhbWJkYSRpbXBsJGEwZmIzNGQ0JDF0APcoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7dAC4KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAfc3IAOW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uUHJvcGVydGllc888WWObqPmzAgADTAANY3VycmVudFpvbmVJZHQAEkxqYXZhL3RpbWUvWm9uZUlkO0wACm5vd0luc3RhbnR0ABNMamF2YS90aW1lL0luc3RhbnQ7TAAJcXVlcnlUeXBldAAnTG9yZy9vcGVuc2VhcmNoL3NxbC9leGVjdXRvci9RdWVyeVR5cGU7eHBzcgANamF2YS50aW1lLlNlcpVdhLobIkiyDAAAeHB3AggAeHNxAH4AQXcNAgAAAABpCZsWIgNaCHh+cgAlb3JnLm9wZW5zZWFyY2guc3FsLmV4ZWN1dG9yLlF1ZXJ5VHlwZQAAAAAAAAAAEgAAeHEAfgAUdAADUFBMfnEAfgATdAAHQk9PTEVBTg==\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"},\"boost\":1.0}},\"\ + _source\":{\"includes\":[\"SearchPhrase\"],\"excludes\":[]}}, needClean=true,\ + \ searchDone=false, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q27.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q27.yaml new file mode 100644 index 00000000000..b009efd7ed4 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q27.yaml @@ -0,0 +1,27 @@ +root: + name: ProjectOperator + description: + fields: "[SearchPhrase]" + children: + - name: LimitOperator + description: + limit: 10 + offset: 0 + children: + - name: ProjectOperator + description: + fields: "[SearchPhrase]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"\ + from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"script\"\ + :{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AAxTZWFyY2hQaHJhc2VzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABlNUUklOR3NyAC9vcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5MaXRlcmFsRXhwcmVzc2lvbkVCLfCMx4IkAgABTAAJZXhwclZhbHVldAApTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt4cHNyAC1vcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5FeHByU3RyaW5nVmFsdWUAQTIlc4kOEwIAAUwABXZhbHVlcQB+AAt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwdAAAeHNyADNvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbk5hbWULqDhNzvZnlwIAAUwADGZ1bmN0aW9uTmFtZXEAfgALeHB0AAIhPXEAfgAJc3IAIWphdmEubGFuZy5pbnZva2UuU2VyaWFsaXplZExhbWJkYW9h0JQsKTaFAgAKSQAOaW1wbE1ldGhvZEtpbmRbAAxjYXB0dXJlZEFyZ3NxAH4AD0wADmNhcHR1cmluZ0NsYXNzdAARTGphdmEvbGFuZy9DbGFzcztMABhmdW5jdGlvbmFsSW50ZXJmYWNlQ2xhc3NxAH4AC0wAHWZ1bmN0aW9uYWxJbnRlcmZhY2VNZXRob2ROYW1lcQB+AAtMACJmdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kU2lnbmF0dXJlcQB+AAtMAAlpbXBsQ2xhc3NxAH4AC0wADmltcGxNZXRob2ROYW1lcQB+AAtMABNpbXBsTWV0aG9kU2lnbmF0dXJlcQB+AAtMABZpbnN0YW50aWF0ZWRNZXRob2RUeXBlcQB+AAt4cAAAAAZ1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAABc3EAfgAhAAAABnVxAH4AJAAAAAFzcQB+ACEAAAAGdXEAfgAkAAAAAHZyAElvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5vcGVyYXRvci5wcmVkaWNhdGUuQmluYXJ5UHJlZGljYXRlT3BlcmF0b3JzAAAAAAAAAAAAAAB4cHQAPW9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb250AAVhcHBseXQAOChMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDspTGphdmEvbGFuZy9PYmplY3Q7dABJb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vb3BlcmF0b3IvcHJlZGljYXRlL0JpbmFyeVByZWRpY2F0ZU9wZXJhdG9yc3QAGmxhbWJkYSRub3RFcXVhbCQ5NTA0OGZjMSQxdAB9KExvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AMXZyADJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTAAAAAAAAAAAAAAAeHBxAH4ALHEAfgAtcQB+AC50ADJvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvbkRTTHQAJWxhbWJkYSRudWxsTWlzc2luZ0hhbmRsaW5nJGE1MDA1MjgxJDF0ALwoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAxcQB+ADN0AD5vcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbnEAfgAtdABKKExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDtxAH4ANHQAFmxhbWJkYSRpbXBsJGEwZmIzNGQ0JDF0APcoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7dAC4KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAfc3IAOW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uUHJvcGVydGllc888WWObqPmzAgADTAANY3VycmVudFpvbmVJZHQAEkxqYXZhL3RpbWUvWm9uZUlkO0wACm5vd0luc3RhbnR0ABNMamF2YS90aW1lL0luc3RhbnQ7TAAJcXVlcnlUeXBldAAnTG9yZy9vcGVuc2VhcmNoL3NxbC9leGVjdXRvci9RdWVyeVR5cGU7eHBzcgANamF2YS50aW1lLlNlcpVdhLobIkiyDAAAeHB3AggAeHNxAH4AQXcNAgAAAABpCZsWJAGi2Hh+cgAlb3JnLm9wZW5zZWFyY2guc3FsLmV4ZWN1dG9yLlF1ZXJ5VHlwZQAAAAAAAAAAEgAAeHEAfgAUdAADUFBMfnEAfgATdAAHQk9PTEVBTg==\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"},\"boost\":1.0}},\"\ + _source\":{\"includes\":[\"SearchPhrase\"],\"excludes\":[]},\"sort\"\ + :[{\"EventTime\":{\"order\":\"asc\",\"missing\":\"_first\"}},{\"\ + SearchPhrase\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, needClean=true,\ + \ searchDone=false, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q28.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q28.yaml new file mode 100644 index 00000000000..c10edef0b2b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q28.yaml @@ -0,0 +1,34 @@ +root: + name: ProjectOperator + description: + fields: "[l, c, CounterID]" + children: + - name: TakeOrderedOperator + description: + limit: 25 + offset: 0 + sortList: + l: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: FilterOperator + description: + conditions: ">(c, 100000)" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"\ + from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"script\":{\"\ + script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AANVUkxzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABlNUUklOR3NyAC9vcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5MaXRlcmFsRXhwcmVzc2lvbkVCLfCMx4IkAgABTAAJZXhwclZhbHVldAApTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt4cHNyAC1vcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5FeHByU3RyaW5nVmFsdWUAQTIlc4kOEwIAAUwABXZhbHVlcQB+AAt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwdAAAeHNyADNvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbk5hbWULqDhNzvZnlwIAAUwADGZ1bmN0aW9uTmFtZXEAfgALeHB0AAIhPXEAfgAJc3IAIWphdmEubGFuZy5pbnZva2UuU2VyaWFsaXplZExhbWJkYW9h0JQsKTaFAgAKSQAOaW1wbE1ldGhvZEtpbmRbAAxjYXB0dXJlZEFyZ3NxAH4AD0wADmNhcHR1cmluZ0NsYXNzdAARTGphdmEvbGFuZy9DbGFzcztMABhmdW5jdGlvbmFsSW50ZXJmYWNlQ2xhc3NxAH4AC0wAHWZ1bmN0aW9uYWxJbnRlcmZhY2VNZXRob2ROYW1lcQB+AAtMACJmdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kU2lnbmF0dXJlcQB+AAtMAAlpbXBsQ2xhc3NxAH4AC0wADmltcGxNZXRob2ROYW1lcQB+AAtMABNpbXBsTWV0aG9kU2lnbmF0dXJlcQB+AAtMABZpbnN0YW50aWF0ZWRNZXRob2RUeXBlcQB+AAt4cAAAAAZ1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAABc3EAfgAhAAAABnVxAH4AJAAAAAFzcQB+ACEAAAAGdXEAfgAkAAAAAHZyAElvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5vcGVyYXRvci5wcmVkaWNhdGUuQmluYXJ5UHJlZGljYXRlT3BlcmF0b3JzAAAAAAAAAAAAAAB4cHQAPW9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb250AAVhcHBseXQAOChMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDspTGphdmEvbGFuZy9PYmplY3Q7dABJb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vb3BlcmF0b3IvcHJlZGljYXRlL0JpbmFyeVByZWRpY2F0ZU9wZXJhdG9yc3QAGmxhbWJkYSRub3RFcXVhbCQ5NTA0OGZjMSQxdAB9KExvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AMXZyADJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTAAAAAAAAAAAAAAAeHBxAH4ALHEAfgAtcQB+AC50ADJvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvbkRTTHQAJWxhbWJkYSRudWxsTWlzc2luZ0hhbmRsaW5nJGE1MDA1MjgxJDF0ALwoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAxcQB+ADN0AD5vcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbnEAfgAtdABKKExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDtxAH4ANHQAFmxhbWJkYSRpbXBsJGEwZmIzNGQ0JDF0APcoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7dAC4KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAfc3IAOW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uUHJvcGVydGllc888WWObqPmzAgADTAANY3VycmVudFpvbmVJZHQAEkxqYXZhL3RpbWUvWm9uZUlkO0wACm5vd0luc3RhbnR0ABNMamF2YS90aW1lL0luc3RhbnQ7TAAJcXVlcnlUeXBldAAnTG9yZy9vcGVuc2VhcmNoL3NxbC9leGVjdXRvci9RdWVyeVR5cGU7eHBzcgANamF2YS50aW1lLlNlcpVdhLobIkiyDAAAeHB3AggAeHNxAH4AQXcNAgAAAABpCZsWJu1c0Hh+cgAlb3JnLm9wZW5zZWFyY2guc3FsLmV4ZWN1dG9yLlF1ZXJ5VHlwZQAAAAAAAAAAEgAAeHEAfgAUdAADUFBMfnEAfgATdAAHQk9PTEVBTg==\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"},\"boost\":1.0}},\"\ + aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"\ + sources\":[{\"CounterID\":{\"terms\":{\"field\":\"CounterID\",\"\ + missing_bucket\":false,\"order\":\"asc\"}}}]},\"aggregations\":{\"\ + l\":{\"avg\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\ + \",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQyPc501CEBPWwCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAP0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0wAEHZhbCRmdW5jdGlvbk5hbWV0ADVMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25OYW1lO0wAFnZhbCRmdW5jdGlvblByb3BlcnRpZXN0ADtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0wADnZhbCRyZXR1cm5UeXBldAAnTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwclR5cGU7eHIAMG9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkZ1bmN0aW9uRXhwcmVzc2lvbrIqMNPcdWp7AgACTAAJYXJndW1lbnRzcQB+AAFMAAxmdW5jdGlvbk5hbWVxAH4AA3hwc3IAE2phdmEudXRpbC5BcnJheUxpc3R4gdIdmcdhnQMAAUkABHNpemV4cAAAAAF3BAAAAAFzcgAxb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uUmVmZXJlbmNlRXhwcmVzc2lvbqtE71wSB4XWAgAETAAEYXR0cnQAEkxqYXZhL2xhbmcvU3RyaW5nO0wABXBhdGhzcQB+AAFMAAdyYXdQYXRocQB+AAtMAAR0eXBlcQB+AAV4cHQAA1VSTHNyABpqYXZhLnV0aWwuQXJyYXlzJEFycmF5TGlzdNmkPL7NiAbSAgABWwABYXQAE1tMamF2YS9sYW5nL09iamVjdDt4cHVyABNbTGphdmEubGFuZy5TdHJpbmc7rdJW5+kde0cCAAB4cAAAAAFxAH4ADXEAfgANfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAGU1RSSU5HeHNyADNvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbk5hbWULqDhNzvZnlwIAAUwADGZ1bmN0aW9uTmFtZXEAfgALeHB0AAZsZW5ndGhxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AGgAAAAZ1cQB+AB0AAAABc3EAfgAaAAAABnVxAH4AHQAAAAB2cgAwb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24udGV4dC5UZXh0RnVuY3Rpb25zAAAAAAAAAAAAAAB4cHQAO29yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUZ1bmN0aW9udAAFYXBwbHl0ACYoTGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3QAMG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL3RleHQvVGV4dEZ1bmN0aW9uc3QAGGxhbWJkYSRsZW5ndGgkMTJjN2RjNDgkMXQAVChMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAqdnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAlcQB+ACZxAH4AJ3QAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckODc4MDY5YzgkMXQAkShMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AKnEAfgAsdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnEAfgAmdAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDtxAH4ALXQAFmxhbWJkYSRpbXBsJDhkNTg2Y2RjJDF0AMwoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0AI8oTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAYc3IAOW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uUHJvcGVydGllc888WWObqPmzAgADTAANY3VycmVudFpvbmVJZHQAEkxqYXZhL3RpbWUvWm9uZUlkO0wACm5vd0luc3RhbnR0ABNMamF2YS90aW1lL0luc3RhbnQ7TAAJcXVlcnlUeXBldAAnTG9yZy9vcGVuc2VhcmNoL3NxbC9leGVjdXRvci9RdWVyeVR5cGU7eHBzcgANamF2YS50aW1lLlNlcpVdhLobIkiyDAAAeHB3AggAeHNxAH4AOncNAgAAAABpCZsWJu1c0Hh+cgAlb3JnLm9wZW5zZWFyY2guc3FsLmV4ZWN1dG9yLlF1ZXJ5VHlwZQAAAAAAAAAAEgAAeHEAfgAUdAADUFBMfnEAfgATdAAHSU5URUdFUg==\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"c\":{\"value_count\"\ + :{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q29.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q29.yaml new file mode 100644 index 00000000000..e69de29bb2d diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q3.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q3.yaml new file mode 100644 index 00000000000..a17c246ae48 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q3.yaml @@ -0,0 +1,14 @@ +root: + name: ProjectOperator + description: + fields: "[sum(AdvEngineID), count(), avg(ResolutionWidth)]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"aggregations\":{\"sum(AdvEngineID)\":{\"sum\"\ + :{\"field\":\"AdvEngineID\"}},\"count()\":{\"value_count\":{\"field\":\"\ + _index\"}},\"avg(ResolutionWidth)\":{\"avg\":{\"field\":\"ResolutionWidth\"\ + }}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q30.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q30.yaml new file mode 100644 index 00000000000..f9001ab2cfc --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q30.yaml @@ -0,0 +1,308 @@ +root: + name: ProjectOperator + description: + fields: "[sum(ResolutionWidth), sum(ResolutionWidth+1), sum(ResolutionWidth+2),\ + \ sum(ResolutionWidth+3), sum(ResolutionWidth+4), sum(ResolutionWidth+5), sum(ResolutionWidth+6),\ + \ sum(ResolutionWidth+7), sum(ResolutionWidth+8), sum(ResolutionWidth+9), sum(ResolutionWidth+10),\ + \ sum(ResolutionWidth+11), sum(ResolutionWidth+12), sum(ResolutionWidth+13),\ + \ sum(ResolutionWidth+14), sum(ResolutionWidth+15), sum(ResolutionWidth+16),\ + \ sum(ResolutionWidth+17), sum(ResolutionWidth+18), sum(ResolutionWidth+19),\ + \ sum(ResolutionWidth+20), sum(ResolutionWidth+21), sum(ResolutionWidth+22),\ + \ sum(ResolutionWidth+23), sum(ResolutionWidth+24), sum(ResolutionWidth+25),\ + \ sum(ResolutionWidth+26), sum(ResolutionWidth+27), sum(ResolutionWidth+28),\ + \ sum(ResolutionWidth+29), sum(ResolutionWidth+30), sum(ResolutionWidth+31),\ + \ sum(ResolutionWidth+32), sum(ResolutionWidth+33), sum(ResolutionWidth+34),\ + \ sum(ResolutionWidth+35), sum(ResolutionWidth+36), sum(ResolutionWidth+37),\ + \ sum(ResolutionWidth+38), sum(ResolutionWidth+39), sum(ResolutionWidth+40),\ + \ sum(ResolutionWidth+41), sum(ResolutionWidth+42), sum(ResolutionWidth+43),\ + \ sum(ResolutionWidth+44), sum(ResolutionWidth+45), sum(ResolutionWidth+46),\ + \ sum(ResolutionWidth+47), sum(ResolutionWidth+48), sum(ResolutionWidth+49),\ + \ sum(ResolutionWidth+50), sum(ResolutionWidth+51), sum(ResolutionWidth+52),\ + \ sum(ResolutionWidth+53), sum(ResolutionWidth+54), sum(ResolutionWidth+55),\ + \ sum(ResolutionWidth+56), sum(ResolutionWidth+57), sum(ResolutionWidth+58),\ + \ sum(ResolutionWidth+59), sum(ResolutionWidth+60), sum(ResolutionWidth+61),\ + \ sum(ResolutionWidth+62), sum(ResolutionWidth+63), sum(ResolutionWidth+64),\ + \ sum(ResolutionWidth+65), sum(ResolutionWidth+66), sum(ResolutionWidth+67),\ + \ sum(ResolutionWidth+68), sum(ResolutionWidth+69), sum(ResolutionWidth+70),\ + \ sum(ResolutionWidth+71), sum(ResolutionWidth+72), sum(ResolutionWidth+73),\ + \ sum(ResolutionWidth+74), sum(ResolutionWidth+75), sum(ResolutionWidth+76),\ + \ sum(ResolutionWidth+77), sum(ResolutionWidth+78), sum(ResolutionWidth+79),\ + \ sum(ResolutionWidth+80), sum(ResolutionWidth+81), sum(ResolutionWidth+82),\ + \ sum(ResolutionWidth+83), sum(ResolutionWidth+84), sum(ResolutionWidth+85),\ + \ sum(ResolutionWidth+86), sum(ResolutionWidth+87), sum(ResolutionWidth+88),\ + \ sum(ResolutionWidth+89)]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"aggregations\":{\"sum(ResolutionWidth)\":{\"\ + sum\":{\"field\":\"ResolutionWidth\"}},\"sum(ResolutionWidth+1)\":{\"sum\"\ + :{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\ + \"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAAF4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+2)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAAJ4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+3)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAAN4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+4)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAAR4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+5)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAAV4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+6)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAAZ4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+7)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAAd4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+8)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAAh4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+9)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAAl4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+10)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAAp4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+11)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAAt4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+12)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAAx4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+13)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAA14c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+14)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAA54c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+15)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAA94c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+16)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAABB4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+17)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAABF4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+18)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAABJ4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+19)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAABN4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+20)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAABR4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+21)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAABV4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+22)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAABZ4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+23)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAABd4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+24)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAABh4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+25)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAABl4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+26)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAABp4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+27)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAABt4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+28)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAABx4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+29)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAB14c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+30)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAB54c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+31)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAB94c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+32)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAACB4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+33)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAACF4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+34)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAACJ4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+35)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAACN4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+36)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAACR4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+37)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAACV4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+38)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAACZ4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+39)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAACd4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+40)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAACh4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+41)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAACl4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+42)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAACp4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+43)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAACt4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+44)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAACx4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+45)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAC14c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+46)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAC54c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+47)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAC94c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+48)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAADB4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+49)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAADF4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+50)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAADJ4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+51)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAADN4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+52)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAADR4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+53)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAADV4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+54)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAADZ4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+55)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAADd4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+56)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAADh4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+57)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAADl4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+58)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAADp4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+59)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAADt4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+60)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAADx4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+61)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAD14c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+62)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAD54c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+63)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAD94c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+64)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAEB4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+65)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAEF4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+66)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAEJ4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+67)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAEN4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+68)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAER4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+69)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAEV4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+70)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAEZ4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+71)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAEd4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+72)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAEh4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+73)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAEl4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+74)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAEp4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+75)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAEt4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+76)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAEx4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+77)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAE14c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+78)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAE54c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+79)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAE94c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+80)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAFB4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+81)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAFF4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+82)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAFJ4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+83)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAFN4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+84)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAFR4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+85)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAFV4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+86)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAFZ4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+87)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAFd4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+88)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAFh4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(ResolutionWidth+89)\"\ + :{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\ + \":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AA9SZXNvbHV0aW9uV2lkdGhzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAFl4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDN0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxYuiyVYeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdJTlRFR0VS\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"}}}}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q31.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q31.yaml new file mode 100644 index 00000000000..22b616bdaef --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q31.yaml @@ -0,0 +1,29 @@ +root: + name: ProjectOperator + description: + fields: "[c, sum(IsRefresh), avg(ResolutionWidth), SearchEngineID, ClientIP]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 0 + sortList: + c: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"\ + source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AAxTZWFyY2hQaHJhc2VzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABlNUUklOR3NyAC9vcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5MaXRlcmFsRXhwcmVzc2lvbkVCLfCMx4IkAgABTAAJZXhwclZhbHVldAApTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt4cHNyAC1vcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5FeHByU3RyaW5nVmFsdWUAQTIlc4kOEwIAAUwABXZhbHVlcQB+AAt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwdAAAeHNyADNvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbk5hbWULqDhNzvZnlwIAAUwADGZ1bmN0aW9uTmFtZXEAfgALeHB0AAIhPXEAfgAJc3IAIWphdmEubGFuZy5pbnZva2UuU2VyaWFsaXplZExhbWJkYW9h0JQsKTaFAgAKSQAOaW1wbE1ldGhvZEtpbmRbAAxjYXB0dXJlZEFyZ3NxAH4AD0wADmNhcHR1cmluZ0NsYXNzdAARTGphdmEvbGFuZy9DbGFzcztMABhmdW5jdGlvbmFsSW50ZXJmYWNlQ2xhc3NxAH4AC0wAHWZ1bmN0aW9uYWxJbnRlcmZhY2VNZXRob2ROYW1lcQB+AAtMACJmdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kU2lnbmF0dXJlcQB+AAtMAAlpbXBsQ2xhc3NxAH4AC0wADmltcGxNZXRob2ROYW1lcQB+AAtMABNpbXBsTWV0aG9kU2lnbmF0dXJlcQB+AAtMABZpbnN0YW50aWF0ZWRNZXRob2RUeXBlcQB+AAt4cAAAAAZ1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAABc3EAfgAhAAAABnVxAH4AJAAAAAFzcQB+ACEAAAAGdXEAfgAkAAAAAHZyAElvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5vcGVyYXRvci5wcmVkaWNhdGUuQmluYXJ5UHJlZGljYXRlT3BlcmF0b3JzAAAAAAAAAAAAAAB4cHQAPW9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb250AAVhcHBseXQAOChMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDspTGphdmEvbGFuZy9PYmplY3Q7dABJb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vb3BlcmF0b3IvcHJlZGljYXRlL0JpbmFyeVByZWRpY2F0ZU9wZXJhdG9yc3QAGmxhbWJkYSRub3RFcXVhbCQ5NTA0OGZjMSQxdAB9KExvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AMXZyADJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTAAAAAAAAAAAAAAAeHBxAH4ALHEAfgAtcQB+AC50ADJvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvbkRTTHQAJWxhbWJkYSRudWxsTWlzc2luZ0hhbmRsaW5nJGE1MDA1MjgxJDF0ALwoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAxcQB+ADN0AD5vcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbnEAfgAtdABKKExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDtxAH4ANHQAFmxhbWJkYSRpbXBsJGEwZmIzNGQ0JDF0APcoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7dAC4KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAfc3IAOW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uUHJvcGVydGllc888WWObqPmzAgADTAANY3VycmVudFpvbmVJZHQAEkxqYXZhL3RpbWUvWm9uZUlkO0wACm5vd0luc3RhbnR0ABNMamF2YS90aW1lL0luc3RhbnQ7TAAJcXVlcnlUeXBldAAnTG9yZy9vcGVuc2VhcmNoL3NxbC9leGVjdXRvci9RdWVyeVR5cGU7eHBzcgANamF2YS50aW1lLlNlcpVdhLobIkiyDAAAeHB3AggAeHNxAH4AQXcNAgAAAABpCZsWM/QvqHh+cgAlb3JnLm9wZW5zZWFyY2guc3FsLmV4ZWN1dG9yLlF1ZXJ5VHlwZQAAAAAAAAAAEgAAeHEAfgAUdAADUFBMfnEAfgATdAAHQk9PTEVBTg==\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"},\"boost\":1.0}},\"\ + aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"\ + sources\":[{\"SearchEngineID\":{\"terms\":{\"field\":\"SearchEngineID\"\ + ,\"missing_bucket\":false,\"order\":\"asc\"}}},{\"ClientIP\":{\"terms\"\ + :{\"field\":\"ClientIP\",\"missing_bucket\":false,\"order\":\"asc\"\ + }}}]},\"aggregations\":{\"c\":{\"value_count\":{\"field\":\"_index\"\ + }},\"sum(IsRefresh)\":{\"sum\":{\"field\":\"IsRefresh\"}},\"avg(ResolutionWidth)\"\ + :{\"avg\":{\"field\":\"ResolutionWidth\"}}}}}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q32.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q32.yaml new file mode 100644 index 00000000000..a5e79981532 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q32.yaml @@ -0,0 +1,29 @@ +root: + name: ProjectOperator + description: + fields: "[c, sum(IsRefresh), avg(ResolutionWidth), WatchID, ClientIP]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 0 + sortList: + c: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"\ + source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AAxTZWFyY2hQaHJhc2VzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABlNUUklOR3NyAC9vcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5MaXRlcmFsRXhwcmVzc2lvbkVCLfCMx4IkAgABTAAJZXhwclZhbHVldAApTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt4cHNyAC1vcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5FeHByU3RyaW5nVmFsdWUAQTIlc4kOEwIAAUwABXZhbHVlcQB+AAt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwdAAAeHNyADNvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbk5hbWULqDhNzvZnlwIAAUwADGZ1bmN0aW9uTmFtZXEAfgALeHB0AAIhPXEAfgAJc3IAIWphdmEubGFuZy5pbnZva2UuU2VyaWFsaXplZExhbWJkYW9h0JQsKTaFAgAKSQAOaW1wbE1ldGhvZEtpbmRbAAxjYXB0dXJlZEFyZ3NxAH4AD0wADmNhcHR1cmluZ0NsYXNzdAARTGphdmEvbGFuZy9DbGFzcztMABhmdW5jdGlvbmFsSW50ZXJmYWNlQ2xhc3NxAH4AC0wAHWZ1bmN0aW9uYWxJbnRlcmZhY2VNZXRob2ROYW1lcQB+AAtMACJmdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kU2lnbmF0dXJlcQB+AAtMAAlpbXBsQ2xhc3NxAH4AC0wADmltcGxNZXRob2ROYW1lcQB+AAtMABNpbXBsTWV0aG9kU2lnbmF0dXJlcQB+AAtMABZpbnN0YW50aWF0ZWRNZXRob2RUeXBlcQB+AAt4cAAAAAZ1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAABc3EAfgAhAAAABnVxAH4AJAAAAAFzcQB+ACEAAAAGdXEAfgAkAAAAAHZyAElvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5vcGVyYXRvci5wcmVkaWNhdGUuQmluYXJ5UHJlZGljYXRlT3BlcmF0b3JzAAAAAAAAAAAAAAB4cHQAPW9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb250AAVhcHBseXQAOChMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDspTGphdmEvbGFuZy9PYmplY3Q7dABJb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vb3BlcmF0b3IvcHJlZGljYXRlL0JpbmFyeVByZWRpY2F0ZU9wZXJhdG9yc3QAGmxhbWJkYSRub3RFcXVhbCQ5NTA0OGZjMSQxdAB9KExvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AMXZyADJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTAAAAAAAAAAAAAAAeHBxAH4ALHEAfgAtcQB+AC50ADJvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvbkRTTHQAJWxhbWJkYSRudWxsTWlzc2luZ0hhbmRsaW5nJGE1MDA1MjgxJDF0ALwoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAxcQB+ADN0AD5vcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbnEAfgAtdABKKExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDtxAH4ANHQAFmxhbWJkYSRpbXBsJGEwZmIzNGQ0JDF0APcoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7dAC4KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAfc3IAOW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uUHJvcGVydGllc888WWObqPmzAgADTAANY3VycmVudFpvbmVJZHQAEkxqYXZhL3RpbWUvWm9uZUlkO0wACm5vd0luc3RhbnR0ABNMamF2YS90aW1lL0luc3RhbnQ7TAAJcXVlcnlUeXBldAAnTG9yZy9vcGVuc2VhcmNoL3NxbC9leGVjdXRvci9RdWVyeVR5cGU7eHBzcgANamF2YS50aW1lLlNlcpVdhLobIkiyDAAAeHB3AggAeHNxAH4AQXcNAgAAAABpCZsWNXsCcHh+cgAlb3JnLm9wZW5zZWFyY2guc3FsLmV4ZWN1dG9yLlF1ZXJ5VHlwZQAAAAAAAAAAEgAAeHEAfgAUdAADUFBMfnEAfgATdAAHQk9PTEVBTg==\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"},\"boost\":1.0}},\"\ + aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"\ + sources\":[{\"WatchID\":{\"terms\":{\"field\":\"WatchID\",\"missing_bucket\"\ + :false,\"order\":\"asc\"}}},{\"ClientIP\":{\"terms\":{\"field\":\"ClientIP\"\ + ,\"missing_bucket\":false,\"order\":\"asc\"}}}]},\"aggregations\":{\"\ + c\":{\"value_count\":{\"field\":\"_index\"}},\"sum(IsRefresh)\":{\"\ + sum\":{\"field\":\"IsRefresh\"}},\"avg(ResolutionWidth)\":{\"avg\":{\"\ + field\":\"ResolutionWidth\"}}}}}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q33.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q33.yaml new file mode 100644 index 00000000000..4da72f0ce53 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q33.yaml @@ -0,0 +1,27 @@ +root: + name: ProjectOperator + description: + fields: "[c, sum(IsRefresh), avg(ResolutionWidth), WatchID, ClientIP]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 0 + sortList: + c: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\"\ + :{\"composite\":{\"size\":1000,\"sources\":[{\"WatchID\":{\"terms\"\ + :{\"field\":\"WatchID\",\"missing_bucket\":false,\"order\":\"asc\"}}},{\"\ + ClientIP\":{\"terms\":{\"field\":\"ClientIP\",\"missing_bucket\":false,\"\ + order\":\"asc\"}}}]},\"aggregations\":{\"c\":{\"value_count\":{\"field\"\ + :\"_index\"}},\"sum(IsRefresh)\":{\"sum\":{\"field\":\"IsRefresh\"}},\"\ + avg(ResolutionWidth)\":{\"avg\":{\"field\":\"ResolutionWidth\"}}}}}},\ + \ needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q34.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q34.yaml new file mode 100644 index 00000000000..d5efb6977f5 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q34.yaml @@ -0,0 +1,24 @@ +root: + name: ProjectOperator + description: + fields: "[c, URL]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 0 + sortList: + c: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\"\ + :{\"composite\":{\"size\":1000,\"sources\":[{\"URL\":{\"terms\":{\"\ + field\":\"URL\",\"missing_bucket\":false,\"order\":\"asc\"}}}]},\"aggregations\"\ + :{\"c\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q35.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q35.yaml new file mode 100644 index 00000000000..9938262d6c3 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q35.yaml @@ -0,0 +1,31 @@ +root: + name: ProjectOperator + description: + fields: "[c, const, URL]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 0 + sortList: + c: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: AggregationOperator + description: + aggregators: "[c]" + groupBy: "[const, URL]" + children: + - name: OpenSearchEvalOperator + description: + expressions: + const: "1" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"\ + from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true,\ + \ searchDone=false, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q36.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q36.yaml new file mode 100644 index 00000000000..60f1883eefc --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q36.yaml @@ -0,0 +1,33 @@ +root: + name: ProjectOperator + description: + fields: "[c, ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 0 + sortList: + c: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: AggregationOperator + description: + aggregators: "[c]" + groupBy: "[ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3]" + children: + - name: OpenSearchEvalOperator + description: + expressions: + ClientIP - 2: "-(ClientIP, 2)" + ClientIP - 1: "-(ClientIP, 1)" + ClientIP - 3: "-(ClientIP, 3)" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"\ + from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true,\ + \ searchDone=false, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q37.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q37.yaml new file mode 100644 index 00000000000..0e9daa42151 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q37.yaml @@ -0,0 +1,38 @@ +root: + name: ProjectOperator + description: + fields: "[PageViews, URL]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 0 + sortList: + PageViews: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"\ + bool\":{\"filter\":[{\"bool\":{\"filter\":[{\"bool\":{\"filter\":[{\"\ + bool\":{\"filter\":[{\"term\":{\"CounterID\":{\"value\":62,\"boost\"\ + :1.0}}},{\"range\":{\"EventDate\":{\"from\":\"2013-07-01 00:00:00\"\ + ,\"to\":null,\"include_lower\":true,\"include_upper\":true,\"boost\"\ + :1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},{\"range\":{\"\ + EventDate\":{\"from\":null,\"to\":\"2013-07-31 00:00:00\",\"include_lower\"\ + :true,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},{\"term\":{\"DontCountHits\":{\"value\":0,\"boost\"\ + :1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},{\"term\":{\"\ + IsRefresh\":{\"value\":0,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\ + \":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AANVUkxzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABlNUUklOR3NyAC9vcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5MaXRlcmFsRXhwcmVzc2lvbkVCLfCMx4IkAgABTAAJZXhwclZhbHVldAApTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt4cHNyAC1vcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5FeHByU3RyaW5nVmFsdWUAQTIlc4kOEwIAAUwABXZhbHVlcQB+AAt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwdAAAeHNyADNvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbk5hbWULqDhNzvZnlwIAAUwADGZ1bmN0aW9uTmFtZXEAfgALeHB0AAIhPXEAfgAJc3IAIWphdmEubGFuZy5pbnZva2UuU2VyaWFsaXplZExhbWJkYW9h0JQsKTaFAgAKSQAOaW1wbE1ldGhvZEtpbmRbAAxjYXB0dXJlZEFyZ3NxAH4AD0wADmNhcHR1cmluZ0NsYXNzdAARTGphdmEvbGFuZy9DbGFzcztMABhmdW5jdGlvbmFsSW50ZXJmYWNlQ2xhc3NxAH4AC0wAHWZ1bmN0aW9uYWxJbnRlcmZhY2VNZXRob2ROYW1lcQB+AAtMACJmdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kU2lnbmF0dXJlcQB+AAtMAAlpbXBsQ2xhc3NxAH4AC0wADmltcGxNZXRob2ROYW1lcQB+AAtMABNpbXBsTWV0aG9kU2lnbmF0dXJlcQB+AAtMABZpbnN0YW50aWF0ZWRNZXRob2RUeXBlcQB+AAt4cAAAAAZ1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAABc3EAfgAhAAAABnVxAH4AJAAAAAFzcQB+ACEAAAAGdXEAfgAkAAAAAHZyAElvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5vcGVyYXRvci5wcmVkaWNhdGUuQmluYXJ5UHJlZGljYXRlT3BlcmF0b3JzAAAAAAAAAAAAAAB4cHQAPW9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb250AAVhcHBseXQAOChMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDspTGphdmEvbGFuZy9PYmplY3Q7dABJb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vb3BlcmF0b3IvcHJlZGljYXRlL0JpbmFyeVByZWRpY2F0ZU9wZXJhdG9yc3QAGmxhbWJkYSRub3RFcXVhbCQ5NTA0OGZjMSQxdAB9KExvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AMXZyADJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTAAAAAAAAAAAAAAAeHBxAH4ALHEAfgAtcQB+AC50ADJvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvbkRTTHQAJWxhbWJkYSRudWxsTWlzc2luZ0hhbmRsaW5nJGE1MDA1MjgxJDF0ALwoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAxcQB+ADN0AD5vcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbnEAfgAtdABKKExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDtxAH4ANHQAFmxhbWJkYSRpbXBsJGEwZmIzNGQ0JDF0APcoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUJpRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7dAC4KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAfc3IAOW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uUHJvcGVydGllc888WWObqPmzAgADTAANY3VycmVudFpvbmVJZHQAEkxqYXZhL3RpbWUvWm9uZUlkO0wACm5vd0luc3RhbnR0ABNMamF2YS90aW1lL0luc3RhbnQ7TAAJcXVlcnlUeXBldAAnTG9yZy9vcGVuc2VhcmNoL3NxbC9leGVjdXRvci9RdWVyeVR5cGU7eHBzcgANamF2YS50aW1lLlNlcpVdhLobIkiyDAAAeHB3AggAeHNxAH4AQXcNAgAAAABpCZsXAiGEWHh+cgAlb3JnLm9wZW5zZWFyY2guc3FsLmV4ZWN1dG9yLlF1ZXJ5VHlwZQAAAAAAAAAAEgAAeHEAfgAUdAADUFBMfnEAfgATdAAHQk9PTEVBTg==\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"},\"boost\":1.0}}],\"\ + adjust_pure_negative\":true,\"boost\":1.0}},\"aggregations\":{\"composite_buckets\"\ + :{\"composite\":{\"size\":1000,\"sources\":[{\"URL\":{\"terms\":{\"\ + field\":\"URL\",\"missing_bucket\":false,\"order\":\"asc\"}}}]},\"aggregations\"\ + :{\"PageViews\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q38.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q38.yaml new file mode 100644 index 00000000000..6c68381eacc --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q38.yaml @@ -0,0 +1,38 @@ +root: + name: ProjectOperator + description: + fields: "[PageViews, Title]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 0 + sortList: + PageViews: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"\ + bool\":{\"filter\":[{\"bool\":{\"filter\":[{\"bool\":{\"filter\":[{\"\ + bool\":{\"filter\":[{\"term\":{\"CounterID\":{\"value\":62,\"boost\"\ + :1.0}}},{\"range\":{\"EventDate\":{\"from\":\"2013-07-01 00:00:00\"\ + ,\"to\":null,\"include_lower\":true,\"include_upper\":true,\"boost\"\ + :1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},{\"range\":{\"\ + EventDate\":{\"from\":null,\"to\":\"2013-07-31 00:00:00\",\"include_lower\"\ + :true,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},{\"term\":{\"DontCountHits\":{\"value\":0,\"boost\"\ + :1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},{\"term\":{\"\ + IsRefresh\":{\"value\":0,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\ + \":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AAVUaXRsZXNyABpqYXZhLnV0aWwuQXJyYXlzJEFycmF5TGlzdNmkPL7NiAbSAgABWwABYXQAE1tMamF2YS9sYW5nL09iamVjdDt4cHVyABNbTGphdmEubGFuZy5TdHJpbmc7rdJW5+kde0cCAAB4cAAAAAFxAH4ADXEAfgANfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAGU1RSSU5Hc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJTdHJpbmdWYWx1ZQBBMiVziQ4TAgABTAAFdmFsdWVxAH4AC3hyAC9vcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJWYWx1ZclrtXYGFESKAgAAeHB0AAB4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAiE9cQB+AAlzcgAhamF2YS5sYW5nLmludm9rZS5TZXJpYWxpemVkTGFtYmRhb2HQlCwpNoUCAApJAA5pbXBsTWV0aG9kS2luZFsADGNhcHR1cmVkQXJnc3EAfgAPTAAOY2FwdHVyaW5nQ2xhc3N0ABFMamF2YS9sYW5nL0NsYXNzO0wAGGZ1bmN0aW9uYWxJbnRlcmZhY2VDbGFzc3EAfgALTAAdZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZE5hbWVxAH4AC0wAImZ1bmN0aW9uYWxJbnRlcmZhY2VNZXRob2RTaWduYXR1cmVxAH4AC0wACWltcGxDbGFzc3EAfgALTAAOaW1wbE1ldGhvZE5hbWVxAH4AC0wAE2ltcGxNZXRob2RTaWduYXR1cmVxAH4AC0wAFmluc3RhbnRpYXRlZE1ldGhvZFR5cGVxAH4AC3hwAAAABnVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAFzcQB+ACEAAAAGdXEAfgAkAAAAAXNxAH4AIQAAAAZ1cQB+ACQAAAAAdnIASW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLm9wZXJhdG9yLnByZWRpY2F0ZS5CaW5hcnlQcmVkaWNhdGVPcGVyYXRvcnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AElvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9wcmVkaWNhdGUvQmluYXJ5UHJlZGljYXRlT3BlcmF0b3JzdAAabGFtYmRhJG5vdEVxdWFsJDk1MDQ4ZmMxJDF0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAxdnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAscQB+AC1xAH4ALnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADFxAH4AM3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+AC10AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA0dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+AB9zcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBBdw0CAAAAAGkJmxcDtwEIeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdCT09MRUFO\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"},\"boost\":1.0}}],\"\ + adjust_pure_negative\":true,\"boost\":1.0}},\"aggregations\":{\"composite_buckets\"\ + :{\"composite\":{\"size\":1000,\"sources\":[{\"Title\":{\"terms\":{\"\ + field\":\"Title\",\"missing_bucket\":false,\"order\":\"asc\"}}}]},\"\ + aggregations\":{\"PageViews\":{\"value_count\":{\"field\":\"_index\"\ + }}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q39.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q39.yaml new file mode 100644 index 00000000000..fbce4c00b5e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q39.yaml @@ -0,0 +1,38 @@ +root: + name: ProjectOperator + description: + fields: "[PageViews, URL]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 1000 + sortList: + PageViews: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"\ + bool\":{\"filter\":[{\"bool\":{\"filter\":[{\"bool\":{\"filter\":[{\"\ + bool\":{\"filter\":[{\"term\":{\"CounterID\":{\"value\":62,\"boost\"\ + :1.0}}},{\"range\":{\"EventDate\":{\"from\":\"2013-07-01 00:00:00\"\ + ,\"to\":null,\"include_lower\":true,\"include_upper\":true,\"boost\"\ + :1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},{\"range\":{\"\ + EventDate\":{\"from\":null,\"to\":\"2013-07-31 00:00:00\",\"include_lower\"\ + :true,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},{\"term\":{\"IsRefresh\":{\"value\":0,\"boost\"\ + :1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},{\"script\":{\"\ + script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\ + \"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AAZJc0xpbmtzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWF0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AA1xAH4ADX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABVNIT1JUc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAAB4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAiE9cQB+AAlzcgAhamF2YS5sYW5nLmludm9rZS5TZXJpYWxpemVkTGFtYmRhb2HQlCwpNoUCAApJAA5pbXBsTWV0aG9kS2luZFsADGNhcHR1cmVkQXJnc3EAfgAPTAAOY2FwdHVyaW5nQ2xhc3N0ABFMamF2YS9sYW5nL0NsYXNzO0wAGGZ1bmN0aW9uYWxJbnRlcmZhY2VDbGFzc3EAfgALTAAdZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZE5hbWVxAH4AC0wAImZ1bmN0aW9uYWxJbnRlcmZhY2VNZXRob2RTaWduYXR1cmVxAH4AC0wACWltcGxDbGFzc3EAfgALTAAOaW1wbE1ldGhvZE5hbWVxAH4AC0wAE2ltcGxNZXRob2RTaWduYXR1cmVxAH4AC0wAFmluc3RhbnRpYXRlZE1ldGhvZFR5cGVxAH4AC3hwAAAABnVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAFzcQB+ACUAAAAGdXEAfgAoAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAAAdnIASW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLm9wZXJhdG9yLnByZWRpY2F0ZS5CaW5hcnlQcmVkaWNhdGVPcGVyYXRvcnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AElvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9wcmVkaWNhdGUvQmluYXJ5UHJlZGljYXRlT3BlcmF0b3JzdAAabGFtYmRhJG5vdEVxdWFsJDk1MDQ4ZmMxJDF0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGkJmxcFMu7AeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUEx+cQB+ABN0AAdCT09MRUFO\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"},\"boost\":1.0}}],\"\ + adjust_pure_negative\":true,\"boost\":1.0}},{\"term\":{\"IsDownload\"\ + :{\"value\":0,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\"\ + :1.0}},\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\"\ + :1000,\"sources\":[{\"URL\":{\"terms\":{\"field\":\"URL\",\"missing_bucket\"\ + :false,\"order\":\"asc\"}}}]},\"aggregations\":{\"PageViews\":{\"value_count\"\ + :{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q4.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q4.yaml new file mode 100644 index 00000000000..e79ebf27361 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q4.yaml @@ -0,0 +1,12 @@ +root: + name: ProjectOperator + description: + fields: "[avg(UserID)]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"aggregations\":{\"avg(UserID)\":{\"avg\":{\"\ + field\":\"UserID\"}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q40.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q40.yaml new file mode 100644 index 00000000000..be6dc42e772 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q40.yaml @@ -0,0 +1,43 @@ +root: + name: ProjectOperator + description: + fields: "[PageViews, TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 1000 + sortList: + PageViews: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: AggregationOperator + description: + aggregators: "[PageViews]" + groupBy: "[TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst]" + children: + - name: OpenSearchEvalOperator + description: + expressions: + Dst: URL + Src: "CaseClause(whenClauses=[WhenClause(condition=and(=(SearchEngineID,\ + \ 0), =(AdvEngineID, 0)), result=Referer)], defaultResult=\"\"\ + )" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"\ + from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\"\ + :{\"filter\":[{\"bool\":{\"filter\":[{\"bool\":{\"filter\":[{\"\ + term\":{\"CounterID\":{\"value\":62,\"boost\":1.0}}},{\"range\"\ + :{\"EventDate\":{\"from\":\"2013-07-01 00:00:00\",\"to\":null,\"\ + include_lower\":true,\"include_upper\":true,\"boost\":1.0}}}],\"\ + adjust_pure_negative\":true,\"boost\":1.0}},{\"range\":{\"EventDate\"\ + :{\"from\":null,\"to\":\"2013-07-31 00:00:00\",\"include_lower\"\ + :true,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},{\"term\":{\"IsRefresh\":{\"value\":0,\"\ + boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}}},\ + \ needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q41.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q41.yaml new file mode 100644 index 00000000000..d70772862ee --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q41.yaml @@ -0,0 +1,39 @@ +root: + name: ProjectOperator + description: + fields: "[PageViews, URLHash, EventDate]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 100 + sortList: + PageViews: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"\ + bool\":{\"filter\":[{\"bool\":{\"filter\":[{\"bool\":{\"filter\":[{\"\ + bool\":{\"filter\":[{\"term\":{\"CounterID\":{\"value\":62,\"boost\"\ + :1.0}}},{\"range\":{\"EventDate\":{\"from\":\"2013-07-01 00:00:00\"\ + ,\"to\":null,\"include_lower\":true,\"include_upper\":true,\"boost\"\ + :1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},{\"range\":{\"\ + EventDate\":{\"from\":null,\"to\":\"2013-07-31 00:00:00\",\"include_lower\"\ + :true,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},{\"term\":{\"IsRefresh\":{\"value\":0,\"boost\"\ + :1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},{\"bool\":{\"\ + should\":[{\"term\":{\"TraficSourceID\":{\"value\":-1,\"boost\":1.0}}},{\"\ + term\":{\"TraficSourceID\":{\"value\":6,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},{\"\ + term\":{\"RefererHash\":{\"value\":3594120000172545465,\"boost\":1.0}}}],\"\ + adjust_pure_negative\":true,\"boost\":1.0}},\"aggregations\":{\"composite_buckets\"\ + :{\"composite\":{\"size\":1000,\"sources\":[{\"URLHash\":{\"terms\"\ + :{\"field\":\"URLHash\",\"missing_bucket\":false,\"order\":\"asc\"}}},{\"\ + EventDate\":{\"terms\":{\"field\":\"EventDate\",\"missing_bucket\":false,\"\ + value_type\":\"long\",\"order\":\"asc\"}}}]},\"aggregations\":{\"PageViews\"\ + :{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q42.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q42.yaml new file mode 100644 index 00000000000..acbba07bc9c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q42.yaml @@ -0,0 +1,38 @@ +root: + name: ProjectOperator + description: + fields: "[PageViews, WindowClientWidth, WindowClientHeight]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 10000 + sortList: + PageViews: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"\ + bool\":{\"filter\":[{\"bool\":{\"filter\":[{\"bool\":{\"filter\":[{\"\ + bool\":{\"filter\":[{\"term\":{\"CounterID\":{\"value\":62,\"boost\"\ + :1.0}}},{\"range\":{\"EventDate\":{\"from\":\"2013-07-01 00:00:00\"\ + ,\"to\":null,\"include_lower\":true,\"include_upper\":true,\"boost\"\ + :1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},{\"range\":{\"\ + EventDate\":{\"from\":null,\"to\":\"2013-07-31 00:00:00\",\"include_lower\"\ + :true,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},{\"term\":{\"IsRefresh\":{\"value\":0,\"boost\"\ + :1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},{\"term\":{\"\ + DontCountHits\":{\"value\":0,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},{\"term\":{\"URLHash\":{\"value\":2868770270353813622,\"\ + boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"aggregations\"\ + :{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"\ + WindowClientWidth\":{\"terms\":{\"field\":\"WindowClientWidth\",\"missing_bucket\"\ + :false,\"order\":\"asc\"}}},{\"WindowClientHeight\":{\"terms\":{\"field\"\ + :\"WindowClientHeight\",\"missing_bucket\":false,\"order\":\"asc\"}}}]},\"\ + aggregations\":{\"PageViews\":{\"value_count\":{\"field\":\"_index\"\ + }}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q43.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q43.yaml new file mode 100644 index 00000000000..b0b6912da8d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q43.yaml @@ -0,0 +1,42 @@ +root: + name: ProjectOperator + description: + fields: "[PageViews, M]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 1000 + sortList: + M: + sortOrder: ASC + nullOrder: NULL_FIRST + children: + - name: AggregationOperator + description: + aggregators: "[PageViews]" + groupBy: "[M]" + children: + - name: OpenSearchEvalOperator + description: + expressions: + M: "date_format(EventTime, \"%Y-%m-%d %H:00:00\")" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"\ + from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\"\ + :{\"filter\":[{\"bool\":{\"filter\":[{\"bool\":{\"filter\":[{\"\ + bool\":{\"filter\":[{\"term\":{\"CounterID\":{\"value\":62,\"\ + boost\":1.0}}},{\"range\":{\"EventDate\":{\"from\":\"2013-07-01\ + \ 00:00:00\",\"to\":null,\"include_lower\":true,\"include_upper\"\ + :true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\"\ + :1.0}},{\"range\":{\"EventDate\":{\"from\":null,\"to\":\"2013-07-15\ + \ 00:00:00\",\"include_lower\":true,\"include_upper\":true,\"\ + boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},{\"\ + term\":{\"IsRefresh\":{\"value\":0,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},{\"term\":{\"DontCountHits\":{\"value\"\ + :0,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\"\ + :1.0}}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q5.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q5.yaml new file mode 100644 index 00000000000..bcaa9940442 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q5.yaml @@ -0,0 +1,12 @@ +root: + name: ProjectOperator + description: + fields: "[dc(UserID)]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"aggregations\":{\"dc(UserID)\":{\"cardinality\"\ + :{\"field\":\"UserID\"}}}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q6.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q6.yaml new file mode 100644 index 00000000000..4e00def0b04 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q6.yaml @@ -0,0 +1,12 @@ +root: + name: ProjectOperator + description: + fields: "[dc(SearchPhrase)]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"aggregations\":{\"dc(SearchPhrase)\":{\"cardinality\"\ + :{\"field\":\"SearchPhrase\"}}}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q7.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q7.yaml new file mode 100644 index 00000000000..8f1350abbd4 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q7.yaml @@ -0,0 +1,13 @@ +root: + name: ProjectOperator + description: + fields: "[min(EventDate), max(EventDate)]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"aggregations\":{\"min(EventDate)\":{\"min\"\ + :{\"field\":\"EventDate\"}},\"max(EventDate)\":{\"max\":{\"field\":\"EventDate\"\ + }}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q8.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q8.yaml new file mode 100644 index 00000000000..2ac33b250eb --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q8.yaml @@ -0,0 +1,24 @@ +root: + name: ProjectOperator + description: + fields: "[count(), AdvEngineID]" + children: + - name: SortOperator + description: + sortList: + count(): + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"\ + source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AAtBZHZFbmdpbmVJRHNyABpqYXZhLnV0aWwuQXJyYXlzJEFycmF5TGlzdNmkPL7NiAbSAgABWwABYXQAE1tMamF2YS9sYW5nL09iamVjdDt4cHVyABNbTGphdmEubGFuZy5TdHJpbmc7rdJW5+kde0cCAAB4cAAAAAFxAH4ADXEAfgANfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAFU0hPUlRzcgAvb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uTGl0ZXJhbEV4cHJlc3Npb25FQi3wjMeCJAIAAUwACWV4cHJWYWx1ZXQAKUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7eHBzcgAub3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuRXhwckludGVnZXJWYWx1ZaZsB5mJt25DAgAAeHIANW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkFic3RyYWN0RXhwck51bWJlclZhbHVlNPRi6sWnMjsCAAFMAAV2YWx1ZXQAEkxqYXZhL2xhbmcvTnVtYmVyO3hyAC9vcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJWYWx1ZclrtXYGFESKAgAAeHBzcgARamF2YS5sYW5nLkludGVnZXIS4qCk94GHOAIAAUkABXZhbHVleHIAEGphdmEubGFuZy5OdW1iZXKGrJUdC5TgiwIAAHhwAAAAAHhzcgAzb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25OYW1lC6g4Tc72Z5cCAAFMAAxmdW5jdGlvbk5hbWVxAH4AC3hwdAACIT1xAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBJb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IucHJlZGljYXRlLkJpbmFyeVByZWRpY2F0ZU9wZXJhdG9ycwAAAAAAAAAAAAAAeHB0AD1vcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9udAAFYXBwbHl0ADgoTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3QASW9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL29wZXJhdG9yL3ByZWRpY2F0ZS9CaW5hcnlQcmVkaWNhdGVPcGVyYXRvcnN0ABpsYW1iZGEkbm90RXF1YWwkOTUwNDhmYzEkMXQAfShMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADV2cgAyb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25EU0wAAAAAAAAAAAAAAHhwcQB+ADBxAH4AMXEAfgAydAAyb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25EU0x0ACVsYW1iZGEkbnVsbE1pc3NpbmdIYW5kbGluZyRhNTAwNTI4MSQxdAC8KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4ANXEAfgA3dAA+b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlVHJpRnVuY3Rpb25xAH4AMXQASihMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDspTGphdmEvbGFuZy9PYmplY3Q7cQB+ADh0ABZsYW1iZGEkaW1wbCRhMGZiMzRkNCQxdAD3KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3QAuChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AI3NyADlvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvblByb3BlcnRpZXPPPFljm6j5swIAA0wADWN1cnJlbnRab25lSWR0ABJMamF2YS90aW1lL1pvbmVJZDtMAApub3dJbnN0YW50dAATTGphdmEvdGltZS9JbnN0YW50O0wACXF1ZXJ5VHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhlY3V0b3IvUXVlcnlUeXBlO3hwc3IADWphdmEudGltZS5TZXKVXYS6GyJIsgwAAHhwdwIIAHhzcQB+AEV3DQIAAAAAaQmbFSnGbXB4fnIAJW9yZy5vcGVuc2VhcmNoLnNxbC5leGVjdXRvci5RdWVyeVR5cGUAAAAAAAAAABIAAHhxAH4AFHQAA1BQTH5xAH4AE3QAB0JPT0xFQU4=\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"},\"boost\":1.0}},\"\ + aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"\ + sources\":[{\"AdvEngineID\":{\"terms\":{\"field\":\"AdvEngineID\",\"\ + missing_bucket\":false,\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\"\ + :{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q9.yaml b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q9.yaml new file mode 100644 index 00000000000..43e4849d7c7 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/clickbench/q9.yaml @@ -0,0 +1,24 @@ +root: + name: ProjectOperator + description: + fields: "[u, RegionID]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 0 + sortList: + u: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=hits, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\"\ + :{\"composite\":{\"size\":1000,\"sources\":[{\"RegionID\":{\"terms\"\ + :{\"field\":\"RegionID\",\"missing_bucket\":false,\"order\":\"asc\"\ + }}}]},\"aggregations\":{\"u\":{\"cardinality\":{\"field\":\"UserID\"\ + }}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/composite_date_histogram_daily.yaml b/integ-test/src/test/resources/expectedOutput/ppl/composite_date_histogram_daily.yaml new file mode 100644 index 00000000000..9a0882dc49a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/composite_date_histogram_daily.yaml @@ -0,0 +1,19 @@ +root: + name: ProjectOperator + description: + fields: "[count(), span(`@timestamp`,1d)]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\"\ + :{\"@timestamp\":{\"from\":1672358400000,\"to\":null,\"include_lower\":true,\"\ + include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\":{\"from\"\ + :null,\"to\":1673092800000,\"include_lower\":true,\"include_upper\":false,\"\ + boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"aggregations\"\ + :{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(`@timestamp`,1d)\"\ + :{\"date_histogram\":{\"field\":\"@timestamp\",\"missing_bucket\":false,\"\ + order\":\"asc\",\"fixed_interval\":\"1d\"}}}]},\"aggregations\":{\"count()\"\ + :{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/composite_terms.yaml b/integ-test/src/test/resources/expectedOutput/ppl/composite_terms.yaml new file mode 100644 index 00000000000..481d9cdd423 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/composite_terms.yaml @@ -0,0 +1,21 @@ +root: + name: ProjectOperator + description: + fields: "[count(), process.name, cloud.region]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\"\ + :{\"@timestamp\":{\"from\":1672617600000,\"to\":null,\"include_lower\":true,\"\ + include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\":{\"from\"\ + :null,\"to\":1672653600000,\"include_lower\":true,\"include_upper\":false,\"\ + boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"aggregations\"\ + :{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"process.name\"\ + :{\"terms\":{\"field\":\"process.name\",\"missing_bucket\":true,\"missing_order\"\ + :\"last\",\"order\":\"desc\"}}},{\"cloud.region\":{\"terms\":{\"field\"\ + :\"cloud.region\",\"missing_bucket\":true,\"missing_order\":\"first\",\"\ + order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"\ + field\":\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/composite_terms_keyword.yaml b/integ-test/src/test/resources/expectedOutput/ppl/composite_terms_keyword.yaml new file mode 100644 index 00000000000..a7f12407647 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/composite_terms_keyword.yaml @@ -0,0 +1,23 @@ +root: + name: ProjectOperator + description: + fields: "[count(), process.name, cloud.region, aws.cloudwatch.log_stream]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\"\ + :{\"@timestamp\":{\"from\":1672617600000,\"to\":null,\"include_lower\":true,\"\ + include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\":{\"from\"\ + :null,\"to\":1672653600000,\"include_lower\":true,\"include_upper\":false,\"\ + boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"aggregations\"\ + :{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"process.name\"\ + :{\"terms\":{\"field\":\"process.name\",\"missing_bucket\":true,\"missing_order\"\ + :\"last\",\"order\":\"desc\"}}},{\"cloud.region\":{\"terms\":{\"field\"\ + :\"cloud.region\",\"missing_bucket\":true,\"missing_order\":\"first\",\"\ + order\":\"asc\"}}},{\"aws.cloudwatch.log_stream\":{\"terms\":{\"field\"\ + :\"aws.cloudwatch.log_stream\",\"missing_bucket\":true,\"missing_order\"\ + :\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\"\ + :{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/date_histogram_hourly_agg.yaml b/integ-test/src/test/resources/expectedOutput/ppl/date_histogram_hourly_agg.yaml new file mode 100644 index 00000000000..72549142297 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/date_histogram_hourly_agg.yaml @@ -0,0 +1,15 @@ +root: + name: ProjectOperator + description: + fields: "[count(), span(`@timestamp`,1h)]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"\ + composite\":{\"size\":1000,\"sources\":[{\"span(`@timestamp`,1h)\":{\"date_histogram\"\ + :{\"field\":\"@timestamp\",\"missing_bucket\":false,\"order\":\"asc\",\"\ + fixed_interval\":\"1h\"}}}]},\"aggregations\":{\"count()\":{\"value_count\"\ + :{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/date_histogram_minute_agg.yaml b/integ-test/src/test/resources/expectedOutput/ppl/date_histogram_minute_agg.yaml new file mode 100644 index 00000000000..be30d2a0801 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/date_histogram_minute_agg.yaml @@ -0,0 +1,19 @@ +root: + name: ProjectOperator + description: + fields: "[count(), span(`@timestamp`,1m)]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\"\ + :{\"@timestamp\":{\"from\":1672531200000,\"to\":null,\"include_lower\":true,\"\ + include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\":{\"from\"\ + :null,\"to\":1672704000000,\"include_lower\":true,\"include_upper\":false,\"\ + boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"aggregations\"\ + :{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(`@timestamp`,1m)\"\ + :{\"date_histogram\":{\"field\":\"@timestamp\",\"missing_bucket\":false,\"\ + order\":\"asc\",\"fixed_interval\":\"1m\"}}}]},\"aggregations\":{\"count()\"\ + :{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/default.yaml b/integ-test/src/test/resources/expectedOutput/ppl/default.yaml new file mode 100644 index 00000000000..23ca821adf6 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/default.yaml @@ -0,0 +1,15 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"agent\",\"process\"\ + ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ + ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_timestamp.yaml new file mode 100644 index 00000000000..ed13e6905cb --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_timestamp.yaml @@ -0,0 +1,16 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"agent\",\"process\"\ + ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ + ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]},\"sort\":[{\"@timestamp\":{\"order\":\"desc\",\"missing\":\"_last\"\ + }}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_timestamp_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_timestamp_can_match_shortcut.yaml new file mode 100644 index 00000000000..e2fc446cdd7 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_timestamp_can_match_shortcut.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"process.name:kernel\"\ + ,\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\"\ + :10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\"\ + :0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\"\ + :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ + ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ + aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\":\"\ + desc\",\"missing\":\"_last\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_timestamp_no_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_timestamp_no_can_match_shortcut.yaml new file mode 100644 index 00000000000..e2fc446cdd7 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_timestamp_no_can_match_shortcut.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"process.name:kernel\"\ + ,\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\"\ + :10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\"\ + :0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\"\ + :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ + ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ + aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\":\"\ + desc\",\"missing\":\"_last\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_with_after_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_with_after_timestamp.yaml new file mode 100644 index 00000000000..ed13e6905cb --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_with_after_timestamp.yaml @@ -0,0 +1,16 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"agent\",\"process\"\ + ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ + ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]},\"sort\":[{\"@timestamp\":{\"order\":\"desc\",\"missing\":\"_last\"\ + }}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_agg_group_merge.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_agg_group_merge.yaml new file mode 100644 index 00000000000..3a063ea858f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_agg_group_merge.yaml @@ -0,0 +1,24 @@ +root: + name: ProjectOperator + description: + fields: "[count(), age1, age2, age3, age]" + children: + - name: AggregationOperator + description: + aggregators: "[count()]" + groupBy: "[age1, age2, age3, age]" + children: + - name: OpenSearchEvalOperator + description: + expressions: + age3: "10" + age2: "+(age, 10)" + age1: "*(age, 10)" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"},\ + \ needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_agg_with_script.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_agg_with_script.json deleted file mode 100644 index 7af36f9596b..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_agg_with_script.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[sum, len, gender]" - }, - "children": [ - { - "name": "AggregationOperator", - "description": { - "aggregators": "[sum]", - "groupBy": "[len, gender]" - }, - "children": [ - { - "name": "OpenSearchEvalOperator", - "description": { - "expressions": { - "len": "length(gender)" - } - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_bank, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } - ] - } - ] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_agg_with_script.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_agg_with_script.yaml new file mode 100644 index 00000000000..f8ce00e8f62 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_agg_with_script.yaml @@ -0,0 +1,22 @@ +root: + name: ProjectOperator + description: + fields: "[sum, len, gender]" + children: + - name: AggregationOperator + description: + aggregators: "[sum]" + groupBy: "[len, gender]" + children: + - name: OpenSearchEvalOperator + description: + expressions: + len: length(gender) + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_bank,\ + \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"},\ + \ needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_keyword_like_function.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_keyword_like_function.json deleted file mode 100644 index f8d72a5bac6..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_keyword_like_function.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]" - }, - "children": [{ - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"wildcard\":{\"firstname.keyword\":{\"wildcard\":\"*mbe*\",\"case_insensitive\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - }, - "children": [] - }] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_keyword_like_function.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_keyword_like_function.yaml new file mode 100644 index 00000000000..5b950289224 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_keyword_like_function.yaml @@ -0,0 +1,17 @@ +root: + name: ProjectOperator + description: + fields: "[account_number, firstname, address, balance, gender, city, employer,\ + \ state, age, email, lastname]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\"\ + :{\"wildcard\":{\"firstname.keyword\":{\"wildcard\":\"*mbe*\",\"case_insensitive\"\ + :true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"account_number\",\"\ + firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"\ + state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, needClean=true,\ + \ searchDone=false, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.yaml index 7a2395fca4b..15f29f544ae 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.yaml @@ -14,7 +14,7 @@ root: missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"pattern_count\"\ :{\"value_count\":{\"field\":\"_index\"}},\"sample_logs\":{\"top_hits\"\ :{\"from\":0,\"size\":10,\"version\":false,\"seq_no_primary_term\":false,\"\ - explain\":false,\"_source\":{\"includes\":[\"email\"],\"excludes\":[]}}}}}}},\ - \ needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + explain\":false,\"fields\":[{\"field\":\"email\"}]}}}}}}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ \ searchResponse=null)" children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_take.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_take.json deleted file mode 100644 index 5e623b3fd26..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_take.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[take]" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"take\":{\"top_hits\":{\"from\":0,\"size\":2,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_take.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_take.yaml new file mode 100644 index 00000000000..7e4de15c4d4 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_take.yaml @@ -0,0 +1,14 @@ +root: + name: ProjectOperator + description: + fields: "[take]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\"\ + :{\"take\":{\"top_hits\":{\"from\":0,\"size\":2,\"version\":false,\"seq_no_primary_term\"\ + :false,\"explain\":false,\"fields\":[{\"field\":\"firstname\"}]}}}}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_text_like_function.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_text_like_function.json deleted file mode 100644 index 224b2835ab5..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_text_like_function.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]" - }, - "children": [{ - "name": "FilterOperator", - "description": { - "conditions": "like(address, \"%Holmes%\")" - }, - "children": [{ - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - }, - "children": [] - }] - }] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_text_like_function.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_text_like_function.yaml new file mode 100644 index 00000000000..c8af0d94dbe --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_text_like_function.yaml @@ -0,0 +1,17 @@ +root: + name: ProjectOperator + description: + fields: "[account_number, firstname, address, balance, gender, city, employer,\ + \ state, age, email, lastname]" + children: + - name: FilterOperator + description: + conditions: "like(address, \"%Holmes%\")" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true,\ + \ searchDone=false, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/keyword_in_range.yaml b/integ-test/src/test/resources/expectedOutput/ppl/keyword_in_range.yaml new file mode 100644 index 00000000000..2a85a0971ce --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/keyword_in_range.yaml @@ -0,0 +1,25 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"bool\"\ + :{\"filter\":[{\"range\":{\"@timestamp\":{\"from\":1672531200000,\"to\"\ + :null,\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}},{\"\ + range\":{\"@timestamp\":{\"from\":null,\"to\":1672704000000,\"include_lower\"\ + :true,\"include_upper\":false,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},{\"query_string\":{\"query\":\"process.name:kernel\"\ + ,\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\"\ + :10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\"\ + :0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},\"_source\":{\"includes\":[\"agent\",\"process\",\"\ + log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\",\"\ + data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/keyword_terms.yaml b/integ-test/src/test/resources/expectedOutput/ppl/keyword_terms.yaml new file mode 100644 index 00000000000..3031899608b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/keyword_terms.yaml @@ -0,0 +1,25 @@ +root: + name: ProjectOperator + description: + fields: "[station, aws.cloudwatch.log_stream]" + children: + - name: TakeOrderedOperator + description: + limit: 500 + offset: 0 + sortList: + station: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\"\ + :{\"composite\":{\"size\":1000,\"sources\":[{\"aws.cloudwatch.log_stream\"\ + :{\"terms\":{\"field\":\"aws.cloudwatch.log_stream\",\"missing_bucket\"\ + :true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\"\ + :{\"station\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/keyword_terms_low_cardinality.yaml b/integ-test/src/test/resources/expectedOutput/ppl/keyword_terms_low_cardinality.yaml new file mode 100644 index 00000000000..2a05fec4f3e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/keyword_terms_low_cardinality.yaml @@ -0,0 +1,25 @@ +root: + name: ProjectOperator + description: + fields: "[country, aws.cloudwatch.log_stream]" + children: + - name: TakeOrderedOperator + description: + limit: 50 + offset: 0 + sortList: + country: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\"\ + :{\"composite\":{\"size\":1000,\"sources\":[{\"aws.cloudwatch.log_stream\"\ + :{\"terms\":{\"field\":\"aws.cloudwatch.log_stream\",\"missing_bucket\"\ + :true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\"\ + :{\"country\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/multi_terms_keyword.yaml b/integ-test/src/test/resources/expectedOutput/ppl/multi_terms_keyword.yaml new file mode 100644 index 00000000000..80709a3e0f5 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/multi_terms_keyword.yaml @@ -0,0 +1,31 @@ +root: + name: ProjectOperator + description: + fields: "[count(), process.name, cloud.region]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 0 + sortList: + count(): + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"\ + range\":{\"@timestamp\":{\"from\":1672876800000,\"to\":null,\"include_lower\"\ + :true,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\"\ + :{\"from\":null,\"to\":1672894800000,\"include_lower\":true,\"include_upper\"\ + :false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"\ + aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"\ + sources\":[{\"process.name\":{\"terms\":{\"field\":\"process.name\"\ + ,\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"\ + }}},{\"cloud.region\":{\"terms\":{\"field\":\"cloud.region\",\"missing_bucket\"\ + :true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\"\ + :{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/query_string_on_message.yaml b/integ-test/src/test/resources/expectedOutput/ppl/query_string_on_message.yaml new file mode 100644 index 00000000000..20f024800e5 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/query_string_on_message.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"((message:monkey\ + \ OR message:jackal) OR message:bear)\",\"fields\":[],\"type\":\"best_fields\"\ + ,\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\"\ + :true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\"\ + :50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\"\ + :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ + ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ + aws\",\"event\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/query_string_on_message_filtered.yaml b/integ-test/src/test/resources/expectedOutput/ppl/query_string_on_message_filtered.yaml new file mode 100644 index 00000000000..fdd6d08721b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/query_string_on_message_filtered.yaml @@ -0,0 +1,26 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"bool\"\ + :{\"filter\":[{\"range\":{\"@timestamp\":{\"from\":1672704000000,\"to\"\ + :null,\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}},{\"\ + range\":{\"@timestamp\":{\"from\":null,\"to\":1672740000000,\"include_lower\"\ + :true,\"include_upper\":false,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},{\"query_string\":{\"query\":\"monkey jackal bear\"\ + ,\"fields\":[\"message^1.0\"],\"type\":\"best_fields\",\"default_operator\"\ + :\"or\",\"max_determinized_states\":10000,\"enable_position_increments\"\ + :true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\"\ + :50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},\"_source\":{\"includes\":[\"agent\",\"process\",\"\ + log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\",\"\ + data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/query_string_on_message_filtered_sorted_num.yaml b/integ-test/src/test/resources/expectedOutput/ppl/query_string_on_message_filtered_sorted_num.yaml new file mode 100644 index 00000000000..4cc79a8db95 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/query_string_on_message_filtered_sorted_num.yaml @@ -0,0 +1,27 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"bool\"\ + :{\"filter\":[{\"range\":{\"@timestamp\":{\"from\":1672704000000,\"to\"\ + :null,\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}},{\"\ + range\":{\"@timestamp\":{\"from\":null,\"to\":1672740000000,\"include_lower\"\ + :true,\"include_upper\":false,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},{\"query_string\":{\"query\":\"monkey jackal bear\"\ + ,\"fields\":[\"message^1.0\"],\"type\":\"best_fields\",\"default_operator\"\ + :\"or\",\"max_determinized_states\":10000,\"enable_position_increments\"\ + :true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\"\ + :50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},\"_source\":{\"includes\":[\"agent\",\"process\",\"\ + log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\",\"\ + data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]},\"sort\":[{\"@timestamp\":{\"order\":\"asc\",\"missing\":\"_first\"\ + }}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range.yaml new file mode 100644 index 00000000000..f9d406b2906 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range.yaml @@ -0,0 +1,19 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\"\ + :{\"@timestamp\":{\"from\":1672531200000,\"to\":null,\"include_lower\":true,\"\ + include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\":{\"from\"\ + :null,\"to\":1672704000000,\"include_lower\":true,\"include_upper\":false,\"\ + boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\"\ + :{\"includes\":[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\"\ + ,\"input\",\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\"\ + ,\"aws\",\"event\"],\"excludes\":[]}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range_agg_1.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range_agg_1.yaml new file mode 100644 index 00000000000..3f99d51799a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range_agg_1.yaml @@ -0,0 +1,28 @@ +root: + name: ProjectOperator + description: + fields: "[count(), range_bucket]" + children: + - name: AggregationOperator + description: + aggregators: "[count()]" + groupBy: "[range_bucket]" + children: + - name: OpenSearchEvalOperator + description: + expressions: + range_bucket: "CaseClause(whenClauses=[WhenClause(condition=<(metrics.size,\ + \ -10), result=\"range_1\"), WhenClause(condition=and(>=(metrics.size,\ + \ -10), <(metrics.size, 10)), result=\"range_2\"), WhenClause(condition=and(>=(metrics.size,\ + \ 10), <(metrics.size, 100)), result=\"range_3\"), WhenClause(condition=and(>=(metrics.size,\ + \ 100), <(metrics.size, 1000)), result=\"range_4\"), WhenClause(condition=and(>=(metrics.size,\ + \ 1000), <(metrics.size, 2000)), result=\"range_5\"), WhenClause(condition=>=(metrics.size,\ + \ 2000), result=\"range_6\")], defaultResult=null)" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"\ + from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true, searchDone=false,\ + \ pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range_agg_2.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range_agg_2.yaml new file mode 100644 index 00000000000..400198d6635 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range_agg_2.yaml @@ -0,0 +1,26 @@ +root: + name: ProjectOperator + description: + fields: "[count(), range_bucket]" + children: + - name: AggregationOperator + description: + aggregators: "[count()]" + groupBy: "[range_bucket]" + children: + - name: OpenSearchEvalOperator + description: + expressions: + range_bucket: "CaseClause(whenClauses=[WhenClause(condition=<(metrics.size,\ + \ 100), result=\"range_1\"), WhenClause(condition=and(>=(metrics.size,\ + \ 100), <(metrics.size, 1000)), result=\"range_2\"), WhenClause(condition=and(>=(metrics.size,\ + \ 1000), <(metrics.size, 2000)), result=\"range_3\"), WhenClause(condition=>=(metrics.size,\ + \ 2000), result=\"range_4\")], defaultResult=null)" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"\ + from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true, searchDone=false,\ + \ pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range_auto_date_histo.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range_auto_date_histo.yaml new file mode 100644 index 00000000000..159c9be49a1 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range_auto_date_histo.yaml @@ -0,0 +1,38 @@ +root: + name: ProjectOperator + description: + fields: "[count(), auto_span, range_bucket]" + children: + - name: SortOperator + description: + sortList: + range_bucket: + sortOrder: ASC + nullOrder: NULL_FIRST + auto_span: + sortOrder: ASC + nullOrder: NULL_FIRST + children: + - name: AggregationOperator + description: + aggregators: "[count()]" + groupBy: "[auto_span, range_bucket]" + children: + - name: OpenSearchEvalOperator + description: + expressions: + range_bucket: "CaseClause(whenClauses=[WhenClause(condition=<(metrics.size,\ + \ -10), result=\"range_1\"), WhenClause(condition=and(>=(metrics.size,\ + \ -10), <(metrics.size, 10)), result=\"range_2\"), WhenClause(condition=and(>=(metrics.size,\ + \ 10), <(metrics.size, 100)), result=\"range_3\"), WhenClause(condition=and(>=(metrics.size,\ + \ 100), <(metrics.size, 1000)), result=\"range_4\"), WhenClause(condition=and(>=(metrics.size,\ + \ 1000), <(metrics.size, 2000)), result=\"range_5\"), WhenClause(condition=>=(metrics.size,\ + \ 2000), result=\"range_6\")], defaultResult=null)" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"\ + from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true,\ + \ searchDone=false, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range_auto_date_histo_with_metrics.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range_auto_date_histo_with_metrics.yaml new file mode 100644 index 00000000000..4f3004af106 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range_auto_date_histo_with_metrics.yaml @@ -0,0 +1,36 @@ +root: + name: ProjectOperator + description: + fields: "[tmin, tavg, tmax, auto_span, range_bucket]" + children: + - name: SortOperator + description: + sortList: + range_bucket: + sortOrder: ASC + nullOrder: NULL_FIRST + auto_span: + sortOrder: ASC + nullOrder: NULL_FIRST + children: + - name: AggregationOperator + description: + aggregators: "[tmin, tavg, tmax]" + groupBy: "[auto_span, range_bucket]" + children: + - name: OpenSearchEvalOperator + description: + expressions: + range_bucket: "CaseClause(whenClauses=[WhenClause(condition=<(metrics.size,\ + \ 100), result=\"range_1\"), WhenClause(condition=and(>=(metrics.size,\ + \ 100), <(metrics.size, 1000)), result=\"range_2\"), WhenClause(condition=and(>=(metrics.size,\ + \ 1000), <(metrics.size, 2000)), result=\"range_3\"), WhenClause(condition=>=(metrics.size,\ + \ 2000), result=\"range_4\")], defaultResult=null)" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"\ + from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true,\ + \ searchDone=false, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range_field_conjunction_big_range_big_term_query.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range_field_conjunction_big_range_big_term_query.yaml new file mode 100644 index 00000000000..2663492036c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range_field_conjunction_big_range_big_term_query.yaml @@ -0,0 +1,21 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"bool\"\ + :{\"filter\":[{\"term\":{\"process.name\":{\"value\":\"systemd\",\"boost\"\ + :1.0}}},{\"range\":{\"metrics.size\":{\"from\":1,\"to\":null,\"include_lower\"\ + :true,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},{\"range\":{\"metrics.size\":{\"from\":null,\"to\"\ + :100,\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}}],\"\ + adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"\ + agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"\ + @timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\"\ + ,\"event\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range_field_conjunction_small_range_big_term_query.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range_field_conjunction_small_range_big_term_query.yaml new file mode 100644 index 00000000000..3365a5b0813 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range_field_conjunction_small_range_big_term_query.yaml @@ -0,0 +1,19 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\"\ + :{\"metrics.size\":{\"from\":20,\"to\":null,\"include_lower\":true,\"include_upper\"\ + :true,\"boost\":1.0}}},{\"range\":{\"metrics.size\":{\"from\":null,\"to\"\ + :30,\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},\"_source\":{\"includes\":[\"agent\",\"process\",\"\ + log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\",\"\ + data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range_field_conjunction_small_range_small_term_query.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range_field_conjunction_small_range_small_term_query.yaml new file mode 100644 index 00000000000..664d6428c7b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range_field_conjunction_small_range_small_term_query.yaml @@ -0,0 +1,21 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"bool\":{\"should\":[{\"term\"\ + :{\"aws.cloudwatch.log_stream\":{\"value\":\"indigodagger\",\"boost\":1.0}}},{\"\ + bool\":{\"filter\":[{\"range\":{\"metrics.size\":{\"from\":10,\"to\":null,\"\ + include_lower\":true,\"include_upper\":true,\"boost\":1.0}}},{\"range\"\ + :{\"metrics.size\":{\"from\":null,\"to\":20,\"include_lower\":true,\"include_upper\"\ + :true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}}],\"\ + adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"\ + agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"\ + @timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\"\ + ,\"event\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range_field_disjunction_big_range_small_term_query.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range_field_disjunction_big_range_small_term_query.yaml new file mode 100644 index 00000000000..641befc2867 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range_field_disjunction_big_range_small_term_query.yaml @@ -0,0 +1,21 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"bool\":{\"should\":[{\"term\"\ + :{\"aws.cloudwatch.log_stream\":{\"value\":\"indigodagger\",\"boost\":1.0}}},{\"\ + bool\":{\"filter\":[{\"range\":{\"metrics.size\":{\"from\":1,\"to\":null,\"\ + include_lower\":true,\"include_upper\":true,\"boost\":1.0}}},{\"range\"\ + :{\"metrics.size\":{\"from\":null,\"to\":100,\"include_lower\":true,\"include_upper\"\ + :true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}}],\"\ + adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"\ + agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"\ + @timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\"\ + ,\"event\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range_numeric.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range_numeric.yaml new file mode 100644 index 00000000000..156f9ced9fe --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range_numeric.yaml @@ -0,0 +1,19 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\"\ + :{\"metrics.size\":{\"from\":20,\"to\":null,\"include_lower\":true,\"include_upper\"\ + :true,\"boost\":1.0}}},{\"range\":{\"metrics.size\":{\"from\":null,\"to\"\ + :200,\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}}],\"\ + adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"\ + agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"\ + @timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\"\ + ,\"event\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range_with_asc_sort.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range_with_asc_sort.yaml new file mode 100644 index 00000000000..05a16cc76cf --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range_with_asc_sort.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\"\ + :{\"@timestamp\":{\"from\":1672531200000,\"to\":null,\"include_lower\":true,\"\ + include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\":{\"from\"\ + :null,\"to\":1673568000000,\"include_lower\":true,\"include_upper\":true,\"\ + boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\"\ + :{\"includes\":[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\"\ + ,\"input\",\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\"\ + ,\"aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\"\ + :\"asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range_with_desc_sort.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range_with_desc_sort.yaml new file mode 100644 index 00000000000..e7322cb282e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range_with_desc_sort.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\"\ + :{\"@timestamp\":{\"from\":1672531200000,\"to\":null,\"include_lower\":true,\"\ + include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\":{\"from\"\ + :null,\"to\":1673568000000,\"include_lower\":true,\"include_upper\":true,\"\ + boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\"\ + :{\"includes\":[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\"\ + ,\"input\",\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\"\ + ,\"aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\"\ + :\"desc\",\"missing\":\"_last\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/scroll.yaml b/integ-test/src/test/resources/expectedOutput/ppl/scroll.yaml new file mode 100644 index 00000000000..23ca821adf6 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/scroll.yaml @@ -0,0 +1,15 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"agent\",\"process\"\ + ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ + ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/sort_keyword_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/ppl/sort_keyword_can_match_shortcut.yaml new file mode 100644 index 00000000000..20b18df0256 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/sort_keyword_can_match_shortcut.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"process.name:kernel\"\ + ,\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\"\ + :10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\"\ + :0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\"\ + :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ + ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ + aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"meta.file\":{\"order\":\"\ + asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/sort_keyword_no_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/ppl/sort_keyword_no_can_match_shortcut.yaml new file mode 100644 index 00000000000..20b18df0256 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/sort_keyword_no_can_match_shortcut.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"process.name:kernel\"\ + ,\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\"\ + :10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\"\ + :0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\"\ + :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ + ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ + aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"meta.file\":{\"order\":\"\ + asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_asc.yaml b/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_asc.yaml new file mode 100644 index 00000000000..9d0f5c0ab0c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_asc.yaml @@ -0,0 +1,16 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"agent\",\"process\"\ + ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ + ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]},\"sort\":[{\"metrics.size\":{\"order\":\"asc\",\"missing\":\"_first\"\ + }}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_asc_with_match.yaml b/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_asc_with_match.yaml new file mode 100644 index 00000000000..3718496568c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_asc_with_match.yaml @@ -0,0 +1,21 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"log.file.path:\\\ + \\/var\\\\/log\\\\/messages\\\\/solarshark\",\"fields\":[],\"type\":\"best_fields\"\ + ,\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\"\ + :true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\"\ + :50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\"\ + :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ + ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ + aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"metrics.size\":{\"order\"\ + :\"asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_desc.yaml b/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_desc.yaml new file mode 100644 index 00000000000..27126b931f0 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_desc.yaml @@ -0,0 +1,16 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"agent\",\"process\"\ + ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ + ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]},\"sort\":[{\"metrics.size\":{\"order\":\"desc\",\"missing\":\"_last\"\ + }}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_desc_with_match.yaml b/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_desc_with_match.yaml new file mode 100644 index 00000000000..a146d0531d5 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_desc_with_match.yaml @@ -0,0 +1,21 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"log.file.path:\\\ + \\/var\\\\/log\\\\/messages\\\\/solarshark\",\"fields\":[],\"type\":\"best_fields\"\ + ,\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\"\ + :true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\"\ + :50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\"\ + :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ + ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ + aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"metrics.size\":{\"order\"\ + :\"desc\",\"missing\":\"_last\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/term.yaml b/integ-test/src/test/resources/expectedOutput/ppl/term.yaml new file mode 100644 index 00000000000..ea9331ffa08 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/term.yaml @@ -0,0 +1,16 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"term\":{\"log.file.path\":{\"\ + value\":\"/var/log/messages/birdknight\",\"boost\":1.0}}},\"_source\":{\"\ + includes\":[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\"\ + ,\"input\",\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\"\ + ,\"aws\",\"event\"],\"excludes\":[]}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/terms_significant_1.yaml b/integ-test/src/test/resources/expectedOutput/ppl/terms_significant_1.yaml new file mode 100644 index 00000000000..09bd047cfb9 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/terms_significant_1.yaml @@ -0,0 +1,27 @@ +root: + name: ProjectOperator + description: + fields: "[count(), aws.cloudwatch.log_stream, process.name]" + children: + - name: LimitOperator + description: + limit: 10 + offset: 0 + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"\ + range\":{\"@timestamp\":{\"from\":1672531200000,\"to\":null,\"include_lower\"\ + :true,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\"\ + :{\"from\":null,\"to\":1672704000000,\"include_lower\":true,\"include_upper\"\ + :false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"\ + aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"\ + sources\":[{\"aws.cloudwatch.log_stream\":{\"terms\":{\"field\":\"aws.cloudwatch.log_stream\"\ + ,\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"\ + }}},{\"process.name\":{\"terms\":{\"field\":\"process.name\",\"missing_bucket\"\ + :true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\"\ + :{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/terms_significant_2.yaml b/integ-test/src/test/resources/expectedOutput/ppl/terms_significant_2.yaml new file mode 100644 index 00000000000..908ab96f04b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/terms_significant_2.yaml @@ -0,0 +1,27 @@ +root: + name: ProjectOperator + description: + fields: "[count(), process.name, aws.cloudwatch.log_stream]" + children: + - name: LimitOperator + description: + limit: 10 + offset: 0 + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"\ + range\":{\"@timestamp\":{\"from\":1672531200000,\"to\":null,\"include_lower\"\ + :true,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\"\ + :{\"from\":null,\"to\":1672704000000,\"include_lower\":true,\"include_upper\"\ + :false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"\ + aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"\ + sources\":[{\"process.name\":{\"terms\":{\"field\":\"process.name\"\ + ,\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"\ + }}},{\"aws.cloudwatch.log_stream\":{\"terms\":{\"field\":\"aws.cloudwatch.log_stream\"\ + ,\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"\ + }}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"\ + }}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/indexDefinitions/events_traffic_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/events_traffic_index_mapping.json new file mode 100644 index 00000000000..d8fe3b4cf55 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/events_traffic_index_mapping.json @@ -0,0 +1,15 @@ +{ + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "packets": { + "type": "integer" + }, + "host": { + "type": "keyword" + } + } + } +} \ No newline at end of file diff --git a/integ-test/src/test/resources/indexDefinitions/locations_type_conflict_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/locations_type_conflict_index_mapping.json new file mode 100644 index 00000000000..98a1bfd07db --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/locations_type_conflict_index_mapping.json @@ -0,0 +1,15 @@ +{ + "mappings": { + "properties": { + "description": { + "type": "text" + }, + "age": { + "type": "text" + }, + "place_id": { + "type": "integer" + } + } + } +} \ No newline at end of file diff --git a/integ-test/src/test/resources/locations_type_conflict.json b/integ-test/src/test/resources/locations_type_conflict.json new file mode 100644 index 00000000000..fd164b3e21c --- /dev/null +++ b/integ-test/src/test/resources/locations_type_conflict.json @@ -0,0 +1,20 @@ +{"index":{"_id":"1"}} +{"description":"Central Park","age":"old","place_id":1001} +{"index":{"_id":"2"}} +{"description":"Times Square","age":"modern","place_id":1002} +{"index":{"_id":"3"}} +{"description":"Brooklyn Bridge","age":"historic","place_id":1003} +{"index":{"_id":"4"}} +{"description":"Empire State Building","age":"1931","place_id":1004} +{"index":{"_id":"5"}} +{"description":"Statue of Liberty","age":"1886","place_id":1005} +{"index":{"_id":"6"}} +{"description":"Grand Central Terminal","age":"vintage","place_id":1006} +{"index":{"_id":"7"}} +{"description":"One World Trade Center","age":"new","place_id":1007} +{"index":{"_id":"8"}} +{"description":"Madison Square Garden","age":"recent","place_id":1008} +{"index":{"_id":"9"}} +{"description":"Wall Street","age":"colonial","place_id":1009} +{"index":{"_id":"10"}} +{"description":"Fifth Avenue","age":"19th century","place_id":1010} diff --git a/integ-test/src/test/resources/time_test_data2.json b/integ-test/src/test/resources/time_test_data2.json new file mode 100644 index 00000000000..db92260798c --- /dev/null +++ b/integ-test/src/test/resources/time_test_data2.json @@ -0,0 +1,40 @@ +{"index":{"_id":"1"}} +{"timestamp":"2025-08-01T04:00:00","value":2001,"category":"E","@timestamp":"2025-08-01T04:00:00"} +{"index":{"_id":"2"}} +{"timestamp":"2025-08-01T02:30:00","value":2002,"category":"F","@timestamp":"2025-08-01T02:30:00"} +{"index":{"_id":"3"}} +{"timestamp":"2025-08-01T01:00:00","value":2003,"category":"E","@timestamp":"2025-08-01T01:00:00"} +{"index":{"_id":"4"}} +{"timestamp":"2025-07-31T22:15:00","value":2004,"category":"F","@timestamp":"2025-07-31T22:15:00"} +{"index":{"_id":"5"}} +{"timestamp":"2025-07-31T20:45:00","value":2005,"category":"E","@timestamp":"2025-07-31T20:45:00"} +{"index":{"_id":"6"}} +{"timestamp":"2025-07-31T18:30:00","value":2006,"category":"F","@timestamp":"2025-07-31T18:30:00"} +{"index":{"_id":"7"}} +{"timestamp":"2025-07-31T16:00:00","value":2007,"category":"E","@timestamp":"2025-07-31T16:00:00"} +{"index":{"_id":"8"}} +{"timestamp":"2025-07-31T14:15:00","value":2008,"category":"F","@timestamp":"2025-07-31T14:15:00"} +{"index":{"_id":"9"}} +{"timestamp":"2025-07-31T12:30:00","value":2009,"category":"E","@timestamp":"2025-07-31T12:30:00"} +{"index":{"_id":"10"}} +{"timestamp":"2025-07-31T10:45:00","value":2010,"category":"F","@timestamp":"2025-07-31T10:45:00"} +{"index":{"_id":"11"}} +{"timestamp":"2025-07-31T08:00:00","value":2011,"category":"E","@timestamp":"2025-07-31T08:00:00"} +{"index":{"_id":"12"}} +{"timestamp":"2025-07-31T06:15:00","value":2012,"category":"F","@timestamp":"2025-07-31T06:15:00"} +{"index":{"_id":"13"}} +{"timestamp":"2025-07-31T04:30:00","value":2013,"category":"E","@timestamp":"2025-07-31T04:30:00"} +{"index":{"_id":"14"}} +{"timestamp":"2025-07-31T02:45:00","value":2014,"category":"F","@timestamp":"2025-07-31T02:45:00"} +{"index":{"_id":"15"}} +{"timestamp":"2025-07-31T01:00:00","value":2015,"category":"E","@timestamp":"2025-07-31T01:00:00"} +{"index":{"_id":"16"}} +{"timestamp":"2025-07-30T23:15:00","value":2016,"category":"F","@timestamp":"2025-07-30T23:15:00"} +{"index":{"_id":"17"}} +{"timestamp":"2025-07-30T21:30:00","value":2017,"category":"E","@timestamp":"2025-07-30T21:30:00"} +{"index":{"_id":"18"}} +{"timestamp":"2025-07-30T19:45:00","value":2018,"category":"F","@timestamp":"2025-07-30T19:45:00"} +{"index":{"_id":"19"}} +{"timestamp":"2025-07-30T18:00:00","value":2019,"category":"E","@timestamp":"2025-07-30T18:00:00"} +{"index":{"_id":"20"}} +{"timestamp":"2025-07-30T16:15:00","value":2020,"category":"F","@timestamp":"2025-07-30T16:15:00"} diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/3459.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/3459.yml new file mode 100644 index 00000000000..c57e33cbd2a --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/3459.yml @@ -0,0 +1,61 @@ +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + + - do: + indices.create: + index: test + body: + mappings: + properties: + parent_field: + properties: + child_field: + type: integer + + - do: + bulk: + index: test + refresh: true + body: + - '{"index": {}}' + - '{ "parent_field": { "child_field": 4 } }' + - '{"index": {}}' + - '{ "parent_field": { "child_field": 3 } }' + - '{"index": {}}' + - '{ "parent_field": { "child_field": 2 } }' + - '{"index": {}}' + - '{ "parent_field": { "child_field": 1 } }' + - '{"index": {}}' + - '{ "parent_field": { "child_field": 5 } }' + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"Access to nested field after field command": + - skip: + features: + - headers + - allowed_warnings + + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test | fields parent_field | sort parent_field.child_field | head 3 + + - match: { total: 3 } + - match: { schema: [ { "name": "parent_field", "type": "struct" } ] } + - match: { datarows: [ [ {"child_field": 1} ], [ {"child_field": 2} ], [ {"child_field": 3} ] ] } diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/3655.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/3655.yml index 57994caa00c..01d1e3fed6c 100644 --- a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/3655.yml +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/3655.yml @@ -44,6 +44,8 @@ teardown: plugins.calcite.enabled : true - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' headers: Content-Type: 'application/json' ppl: @@ -54,6 +56,8 @@ teardown: - match: {"datarows": [["This is a match_only_text field 1"]]} - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' headers: Content-Type: 'application/json' ppl: @@ -66,6 +70,8 @@ teardown: --- "Support match_only_text field type with Calcite disabled": - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' headers: Content-Type: 'application/json' ppl: diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/3946.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/3946.yml new file mode 100644 index 00000000000..d68c5c75835 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/3946.yml @@ -0,0 +1,71 @@ +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: true + plugins.ppl.syntax.legacy.preferred: true + - do: + indices.create: + index: test_divide_settings + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + - do: + bulk: + index: test_divide_settings + refresh: true + body: + - '{"index": {}}' + - '{"id": 1}' + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: false + plugins.ppl.syntax.legacy.preferred: true + - do: + indices.delete: + index: test_divide_settings + +--- +"legacy division retains integer truncation": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_divide_settings | eval a=4/2 | eval b=2/4 | eval c=2/40 | fields a,b,c + - match: { total: 1 } + - match: { datarows: [[2,0,0]] } + +--- +"non-legacy division returns floating values": + - skip: + features: + - headers + - allowed_warnings + - do: + query.settings: + body: + transient: + plugins.ppl.syntax.legacy.preferred: false + - do: + allowed_warnings: [] + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_divide_settings | eval a=4/2 | eval b=2/4 | eval c=2/40 | fields a,b,c + - match: { total: 1 } + - match: { datarows: [[2.0,0.5,0.05]] } diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4201.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4201.yml new file mode 100644 index 00000000000..2ff2277f625 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4201.yml @@ -0,0 +1,112 @@ +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + + - do: + indices.create: + index: test + body: + mappings: + properties: + "@timestamp": + type: date + timestamp: + type: date + size: + type: long + tmin: + type: double + metrics: + type: object + properties: + size: + type: long + tmin: + type: double + + - do: + bulk: + index: test + refresh: true + body: + - '{"index": {}}' + - '{ "@timestamp": "2025-01-01T00:00:00Z", "timestamp": "2025-01-01T00:00:00Z", "size": -20, "tmin": 1.0, "metrics": { "size": -20, "tmin": 1.0 } }' + - '{"index": {}}' + - '{ "@timestamp": "2025-01-01T01:00:00Z", "timestamp": "2025-01-01T01:00:00Z", "size": 5, "tmin": 2.5, "metrics": { "size": 5, "tmin": 2.5 } }' + - '{"index": {}}' + - '{ "@timestamp": "2025-01-01T02:00:00Z", "timestamp": "2025-01-01T02:00:00Z", "size": 50, "tmin": 3.2, "metrics": { "size": 50, "tmin": 3.2 } }' + - '{"index": {}}' + - '{ "@timestamp": "2025-01-01T03:00:00Z", "timestamp": "2025-01-01T03:00:00Z", "size": 500, "tmin": 1.8, "metrics": { "size": 500, "tmin": 1.8 } }' + - '{"index": {}}' + - '{ "@timestamp": "2025-01-01T04:00:00Z", "timestamp": "2025-01-01T04:00:00Z", "size": 1500, "tmin": 4.1, "metrics": { "size": 1500, "tmin": 4.1 } }' + - '{"index": {}}' + - '{ "@timestamp": "2025-01-01T05:00:00Z", "timestamp": "2025-01-01T05:30:00Z", "size": 3000, "tmin": 2.9, "metrics": { "size": 3000, "tmin": 2.9 } }' + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"Test aggregation by range bucket": + - skip: + features: + - headers + - allowed_warnings + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: | + source = test + | eval range_bucket = case( + `metrics.size` < -10, 'range_1', + `metrics.size` >= -10 and `metrics.size` < 10, 'range_2', + `metrics.size` >= 10 and `metrics.size` < 100, 'range_3', + `metrics.size` >= 100 and `metrics.size` < 1000, 'range_4', + `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_5', + `metrics.size` >= 2000, 'range_6' + ) + | stats min(`metrics.tmin`) as tmin, avg(`metrics.size`) as tavg, max(`metrics.size`) as tmax + by range_bucket + + - match: { total: 6 } + - match: { schema: [{"name": "tmin", "type": "double"}, {"name": "tavg", "type": "double"}, {"name": "tmax", "type": "bigint"}, {"name": "range_bucket", "type": "string"}] } + - match: { datarows: [[1.0, -20.0, -20, "range_1"], [2.5, 5.0, 5, "range_2"], [3.2, 50.0, 50, "range_3"], [1.8, 500.0, 500, "range_4"], [4.1, 1500.0, 1500, "range_5"], [2.9, 3000.0, 3000, "range_6"]] } + +--- +"Test aggregation by range bucket and time span": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: | + source = test + | eval range_bucket = case( + `metrics.size` < -10, 'range_1', + `metrics.size` >= -10 and `metrics.size` < 10, 'range_2', + `metrics.size` >= 10 and `metrics.size` < 100, 'range_3', + `metrics.size` >= 100 and `metrics.size` < 1000, 'range_4', + `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_5', + `metrics.size` >= 2000, 'range_6' + ) + | stats min(`metrics.tmin`) as tmin, avg(`metrics.size`) as tavg, max(`metrics.size`) as tmax + by range_bucket, span(`@timestamp`, 1h) + + - match: { total: 6 } + - match: { schema: [{"name": "tmin", "type": "double"}, {"name": "tavg", "type": "double"}, {"name": "tmax", "type": "bigint"}, {"name": "span(`@timestamp`,1h)", "type": "timestamp"}, {"name": "range_bucket", "type": "string"}] } + - match: { datarows: [[1.0, -20.0, -20, "2025-01-01 00:00:00", "range_1"], [2.5, 5.0, 5, "2025-01-01 01:00:00", "range_2"], [3.2, 50.0, 50, "2025-01-01 02:00:00", "range_3"], [1.8, 500.0, 500, "2025-01-01 03:00:00", "range_4"], [4.1, 1500.0, 1500, "2025-01-01 04:00:00", "range_5"], [2.9, 3000.0, 3000, "2025-01-01 05:00:00", "range_6"]] } diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4272.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4272.yml new file mode 100644 index 00000000000..35521ccacf2 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4272.yml @@ -0,0 +1,42 @@ +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"Test validation error for count(eval) with non-boolean expression": + - skip: + features: + - headers + - allowed_warnings + - do: + bulk: + index: test_accounts + refresh: true + body: + - '{"index": {}}' + - '{"age": 25, "name": "John", "balance": 1000}' + - '{"index": {}}' + - '{"age": 30, "name": "Jane", "balance": 2000}' + - '{"index": {}}' + - '{"age": 35, "name": "Bob", "balance": 1500}' + + # Test case: count(eval()) with non-boolean expression should throw validation error + - do: + catch: bad_request + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_accounts | stats count(eval(age)) as cnt + - match: {"$body": "/Condition\\s+expected\\s+a\\s+boolean\\s+type,\\s+but\\s+got\\s+BIGINT/"} diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4339.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4339.yml new file mode 100644 index 00000000000..e958436d1de --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4339.yml @@ -0,0 +1,83 @@ +setup: + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + log: + properties: + url: + properties: + message: + type: text + fields: + keyword: + type: keyword + ignore_above: 256 + time: + type: long + message_alias: + type: alias + path: log.url.message + time_alias: + type: alias + path: log.url.time + + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"dedup struct field": + - skip: + features: + - headers + - allowed_warnings + - do: + bulk: + index: test + refresh: true + body: + - '{"index": {}}' + - '{"log": {"url": {"message": "/e2e/h/zap", "time": 1} } }' + - '{"index": {}}' + - '{"log": {"url": {"message": "/e2e/h/zap", "time": 1} } }' + - '{"index": {}}' + - '{"log": {"url": {"message": "/e2e/h/zap", "time": 2} } }' + - '{"index": {}}' + - '{"log": {"url": {"message": "/e2e/h/zap", "time": 2} } }' + - '{"index": {}}' + - '{"log": {"url": {"message": "/e2e/h/aap", "time": 1} } }' + - '{"index": {}}' + - '{"log": {"url": {"message": "/e2e/h/aap", "time": 1} } }' + - '{"index": {}}' + - '{"log": {"url": {"time": 1} } }' + - '{"index": {}}' + - '{"log": {"url": {"time": 1} } }' + - '{"index": {}}' + - '{"log": {"url": {"time": 2} } }' + - '{"index": {}}' + - '{"log": {"url": {"time": 2} } }' + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: 'source=test | dedup log' + - match: {"total": 5} diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4342.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4342.yml index a3189e7588a..35021cafb98 100644 --- a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4342.yml +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4342.yml @@ -37,6 +37,8 @@ teardown: - '{ "@timestamp" : "2025-09-04T16:15:00.000Z" }' - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' headers: Content-Type: 'application/json' ppl: diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4356.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4356.yml new file mode 100644 index 00000000000..e4a99a268b2 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4356.yml @@ -0,0 +1,218 @@ +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + + - do: + indices.create: + index: log00001 + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + v: + type: text + strnum: + type: keyword + vint: + type: integer + vdouble: + type: double + vboolean: + type: boolean + + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "log00001", "_id": 1}}' + - '{"v": "value=1", "a": 1, "vint": 1, "vdouble": 1.0, "strnum": "1"}' + - '{"index": {"_index": "log00001", "_id": 2}}' + - '{"v": "value=1.5", "a": 2, "vint": 1, "vdouble": 1.5, "strnum": "2"}' + - '{"index": {"_index": "log00001", "_id": 3}}' + - '{"v": "value=true", "a": 3, "vint": 1, "vdouble": 1.0, "vboolean":true, "strnum": "3"}' + - '{"index": {"_index": "log00001", "_id": 4}}' + - '{"v": "value=abcde", "a": 4, "vint": 1, "vdouble": 1.0, "strnum": "malformed"}' + - do: + indices.create: + index: log00002 + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + id: + type: integer + + + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "log00002", "_id": 1}}' + - '{"id": 1}' + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + - do: + indices.delete: + index: log00001 + ignore_unavailable: true + +--- +"Extracted value participate in arithmetic operator": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=log00001 | rex field=v 'value=(?[\\w\\d\\.]*)' | eval m=digits * 10 | eval d=digits/10 | sort a | fields m, d + - match: {"schema": [{"name": "m", "type": "double"}, {"name": "d", "type": "double"}]} + - match: {"datarows": [[10.0, 0.1], [15.0, 0.15], [null, null], [null, null]]} + + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=log00001 | rex field=v 'value=(?[\\w\\d\\.]*)' | eval m=digits + digits, d=digits * digits | sort a | fields m, d + - match: { "schema": [ { "name": "m", "type": "string" }, { "name": "d", "type": "double" } ] } + - match: { "datarows": [ [ "11", 1.0 ], [ "1.51.5", 2.25 ], [ "truetrue", null ], [ "abcdeabcde", null ] ] } + + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=log00002 | eval m="5" - 10 | eval r=round("1.5", 1) | eval f=floor("5.2") | eval c=ceil("5.2") | fields m, r, f, c + - match: { "schema": [ { "name": "m", "type": "double" }, { "name": "r", "type": "double" }, { "name": "f", "type": "double" }, { "name": "c", "type": "double" }] } + - match: { "datarows": [ [ -5.0, 1.5, 5.0, 6.0] ] } + +--- +"Extracted value participate in comparison operator": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=log00001 | rex field=v 'value=(?[\\w\\d\\.]*)' | eval i=digits==vint, d=digits==vdouble, b=digits==vboolean| fields i, d, b + - match: {"schema": [{"name": "i", "type": "boolean"}, {"name": "d", "type": "boolean"}, {"name": "b", "type": "boolean"}]} + - match: {"datarows": [[true,true,null], [false,true,null], [null, null, true], [null, null, null]]} + + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=log00002 | eval e='1000'==1000, en='1000'!=1000, ed='1000'==1000.0, edn='1000'!=1000.0, l='1000'>999, ld='1000'>999.9, i="malformed"==1000 | fields e, en, ed, edn, l, ld, i + - match: {"schema": [{"name": "e", "type": "boolean"}, {"name": "en", "type": "boolean"}, {"name": "ed", "type": "boolean"}, {"name": "edn", "type": "boolean"}, {"name": "l", "type": "boolean"}, {"name": "ld", "type": "boolean"}, {"name": "i", "type": "boolean"}]} + - match: {"datarows": [[true, false, true, false, true, true, null]]} + +--- +"Extracted value participate in string func": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=log00001 | rex field=v 'value=(?[\\w\\d\\.]*)' | eval r=concat('v-', digits) | sort a | fields r + - match: {"schema": [{"name": "r", "type": "string"}]} + - match: {"datarows": [["v-1"], ["v-1.5"], ["v-true"], ["v-abcde"]]} + + +--- +"Extracted value participate in condition func": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=log00001 | rex field=v 'value=(?[\\w\\d\\.]*)' | eval isNull=isnull(digits) | fields isNull + - match: {"schema": [{"name": "isNull", "type": "boolean"}]} + - match: {"datarows": [[false], [false], [false], [false]]} + +--- +"Extracted value participate in aggregation func": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=log00001 | rex field=v 'value=(?[\\w\\d\\.]*)' | stats count(digits) as cnt, sum(digits) as sum, avg(digits) as avg + - match: {"schema": [{"name": "cnt", "type": "bigint"}, {"name": "sum", "type": "double"}, {"name": "avg", "type": "double"}]} + - match: {"datarows": [[4, 2.5, 1.25]]} + + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=log00001 | rex field=v 'value=(?[\\w\\d\\.]*)' | eventstats sum(digits) as sum, count(digits) as cnt| fields a, sum, cnt + - match: { "schema": [ { "name": "a", "type": "bigint" }, { "name": "sum", "type": "double" }, { "name": "cnt", "type": "bigint" } ] } + - match: { "datarows": [ [1, 2.5, 4], [2, 2.5, 4], [3, 2.5, 4], [4, 2.5, 4] ] } + +--- +"Safe cast keyword to int": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=log00001 | stats count(strnum) as cnt, sum(strnum) as sum, avg(strnum) as avg + - match: {"schema": [{"name": "cnt", "type": "bigint"}, {"name": "sum", "type": "double"}, {"name": "avg", "type": "double"}]} +# Notice: Count is calculated on string value, sum and avg are calculated on numeric value, this is why sum/count!=avg + - match: {"datarows": [[4, 6.0, 2.0]]} diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4383.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4383.yml new file mode 100644 index 00000000000..6e0d05a8199 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4383.yml @@ -0,0 +1,64 @@ +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"Append UDT fields should merge schema successfully": + - skip: + features: + - headers + - allowed_warnings + - do: + indices.create: + index: log-test1 + body: + mappings: + properties: + "timestamp": + type: date + - do: + indices.create: + index: log-test2 + body: + mappings: + properties: + "host": + type: ip + + - do: + bulk: + index: log-test1 + refresh: true + body: + - '{"index": {}}' + - '{ "timestamp" : "2025-09-04T16:15:00.000Z" }' + - do: + bulk: + index: log-test2 + refresh: true + body: + - '{"index": {}}' + - '{ "host" : "0.0.0.2" }' + + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=log-test1 | append [ source=log-test2 ] + + - match: { total: 2 } + - match: { datarows: [["2025-09-04 16:15:00", null], [null, "0.0.0.2"]] } diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4391.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4391.yml index 19f9e0d5205..fe465e5005d 100644 --- a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4391.yml +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4391.yml @@ -27,6 +27,8 @@ teardown: - headers - allowed_warnings - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' headers: Content-Type: 'application/json' ppl: diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4407.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4407.yml index 2549b73c570..c6be11da17f 100644 --- a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4407.yml +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4407.yml @@ -27,6 +27,8 @@ teardown: - headers - allowed_warnings - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' headers: Content-Type: 'application/json' ppl: diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4415.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4415.yml new file mode 100644 index 00000000000..fd4fa7629f8 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4415.yml @@ -0,0 +1,44 @@ +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + - do: + bulk: + index: test + refresh: true + body: + - '{"index": {}}' + - '{"num": 11}' + - '{"index": {}}' + - '{"num": 15}' + - '{"index": {}}' + - '{"num": 22}' + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"big decimal literal": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test | bin num bins=3 | stats count() by num + + - match: { total: 2 } + - match: { "schema": [ { "name": "count()", "type": "bigint" }, { "name": "num", "type": "string" }] } + - match: {"datarows": [[2, "10-20"], [1, "20-30"]]} diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4469.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4469.yml new file mode 100644 index 00000000000..56301883f73 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4469.yml @@ -0,0 +1,61 @@ +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + - do: + indices.create: + index: test + body: + mappings: + properties: + "timestamp": + type: date + "status": + type: integer + "client_ip": + type: ip + - do: + bulk: + index: test + refresh: true + body: + - '{"index":{}}' + - '{"datetime":"2025-01-15T00:30:00Z","status":200,"client_ip":"10.0.0.1"}' + - '{"index":{}}' + - '{"datetime":"2025-01-15T02:15:00Z","status":200,"client_ip":"10.0.0.2"}' + - '{"index":{}}' + - '{"datetime":"2025-01-15T10:50:00Z","status":200,"client_ip":"10.0.0.11"}' + - '{"index":{}}' + - '{"datetime":"2025-01-15T23:45:00Z","status":200,"client_ip":"10.0.0.24"}' + + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"agg with script and sort on group key": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test | eval hour_of_day = HOUR(datetime) | stats count() as request_count by hour_of_day | sort hour_of_day asc + + + - match: { total: 4 } + - match: { "schema": [ { "name": "request_count", "type": "bigint" }, { "name": "hour_of_day", "type": "int" }] } + # it got [[1, 0], [1, 10], [1, 2], [1, 23]] without the fix of this issue + - match: {"datarows": [[1, 0], [1, 2], [1, 10], [1, 23]]} diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4481.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4481.yml new file mode 100644 index 00000000000..c416be2bc11 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4481.yml @@ -0,0 +1,61 @@ +"Fix missing keywordsCanBeID": + - skip: + features: + - headers + - allowed_warnings + - do: + indices.create: + index: log-test + body: + mappings: + properties: + "as": + properties: + "field": + properties: + "on": + properties: + "limit": + properties: + "datamodel": + properties: + "overwrite": + properties: + "sed": + properties: + "label": + properties: + "aggregation": + properties: + "brain": + properties: + "simple_pattern": + properties: + "max_match": + properties: + "offset_field": + properties: + "to": + properties: + "millisecond": + type: integer + + - do: + bulk: + index: log-test + refresh: true + body: + - '{"index": {}}' + - '{"as": {"field": {"on": {"limit": {"datamodel": {"overwrite": {"sed": {"label": {"aggregation": {"brain": {"simple_pattern": {"max_match": {"offset_field": {"to": {"millisecond": 1 } } } } } } } } } } } } } } }' + + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=log-test | fields as.field.on.limit.datamodel.overwrite.sed.label.aggregation.brain.simple_pattern.max_match.offset_field.to.millisecond + + - match: { total: 1 } + - length: { datarows: 1 } diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4490.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4490.yml new file mode 100644 index 00000000000..f41f23300df --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4490.yml @@ -0,0 +1,57 @@ +setup: + - do: + indices.create: + index: test + body: + mappings: + properties: + "startTimeMillis": + type: date + format: epoch_millis + + - do: + bulk: + index: test + refresh: true + body: + - '{"index": {"_id": "1"}}' + - '{"startTimeMillis": 539325296000}' + - '{"index": {"_id": "2"}}' + - '{"startTimeMillis": "1715126504378"}' +--- +"handle epoch_millis format field with equal": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test | eval example_time = STR_TO_DATE('1987-02-03 04:34:56', '%Y-%m-%d %H:%i:%S') | where startTimeMillis = example_time | fields startTimeMillis + + - match: { total: 1 } + - match: {"schema": [{"name": "startTimeMillis", "type": "timestamp"}]} + - match: {"datarows": [["1987-02-03 04:34:56"]]} + +--- +"handle epoch_millis format field with comparison": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test | eval example_time = STR_TO_DATE('1987-02-03 04:34:56', '%Y-%m-%d %H:%i:%S') | where startTimeMillis >= example_time | fields startTimeMillis + + - match: { total: 2 } + - match: {"schema": [{"name": "startTimeMillis", "type": "timestamp"}]} + - match: {"datarows": [["1987-02-03 04:34:56"], ["2024-05-08 00:01:44.378"]]} diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4513.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4513.yml new file mode 100644 index 00000000000..a841d84a11b --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4513.yml @@ -0,0 +1,49 @@ +setup: + - do: + bulk: + index: test + refresh: true + body: + - '{"index": {"_id": "1"}}' + - '{"value": "1"}' + - '{"index": {"_id": "2"}}' + - '{"value": "2"}' + - '{"index": {"_id": "3"}}' + - '{"value": "3"}' +--- +"handle agg script push down on metadata field": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test | eval id_int = cast(_id as int) | stats median(id_int) + + - match: { total: 1 } + - match: {"schema": [{"name": "median(id_int)", "type": "int"}]} + - match: {"datarows": [[2]]} + +--- +"handle filter script push down on metadata field": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test | eval id_int = cast(_id as int) | where id_int > 1 + + - match: { total: 2 } + - match: {"schema": [{"name": "value", "type": "string"}, {"name": "id_int", "type": "int"}]} + - match: {"datarows": [["2", 2], ["3", 3]]} diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4527.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4527.yml new file mode 100644 index 00000000000..6733ddeef47 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4527.yml @@ -0,0 +1,52 @@ +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + - do: + indices.create: + index: test_timechart_span_validation + body: + mappings: + properties: + "@timestamp": + type: date_nanos + packets: + type: long + - do: + bulk: + index: test_timechart_span_validation + refresh: true + body: + - '{"index": {}}' + - '{"@timestamp": "2024-01-15T10:30:04.567890123Z", "packets": 100}' + - '{"index": {}}' + - '{"@timestamp": "2024-01-15T10:31:04.567890123Z", "packets": 150}' + - '{"index": {}}' + - '{"@timestamp": "2024-01-15T10:32:04.567890123Z", "packets": 120}' + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"timechart with zero span should return validation error": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + catch: bad_request + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_timechart_span_validation | timechart span=0m per_second(packets) + - match: {"$body": "/Zero\\s+or\\s+negative\\s+time\\s+interval\\s+not\\s+supported/"} diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4529.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4529.yml new file mode 100644 index 00000000000..7c25335de89 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4529.yml @@ -0,0 +1,48 @@ +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + - do: + bulk: + index: test + refresh: true + body: + - '{"index": {}}' + - '{"category":"A","has_flag":true,"value":10}' + - '{"index": {}}' + - '{"category":"B","has_flag":true,"value":20}' + - '{"index": {}}' + - '{"category":"C","has_flag":true,"value":30}' + - '{"index": {}}' + - '{"category":"D","has_flag":false,"value":40}' + - '{"index": {}}' + - '{"category":"E","has_flag":false,"value":50}' + - '{"index": {}}' + - '{"category":"F","has_flag":false,"value":60}' + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"Join with fields": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test | stats COUNT() as cnt by category, has_flag | fields category, has_flag, cnt | join left=L right=R ON L.has_flag = R.has_flag [source=test | stats COUNT() as overall_cnt by has_flag] + + - match: { total: 6 } diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4547.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4547.yml new file mode 100644 index 00000000000..d9970e2564e --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4547.yml @@ -0,0 +1,196 @@ +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + - do: + indices.create: + index: test_wide_schema + body: + - ' + { + "mappings": { + "properties": { + "activity_name": {"type": "keyword"}, + "start_time": {"type": "date"}, + "end_time": {"type": "date"}, + "field_001": {"type": "keyword"}, "field_002": {"type": "keyword"}, "field_003": {"type": "long"}, + "field_004": {"type": "keyword"}, "field_005": {"type": "text"}, "field_006": {"type": "long"}, + "field_007": {"type": "keyword"}, "field_008": {"type": "keyword"}, "field_009": {"type": "long"}, + "field_010": {"type": "text"}, "field_011": {"type": "keyword"}, "field_012": {"type": "long"}, + "field_013": {"type": "keyword"}, "field_014": {"type": "keyword"}, "field_015": {"type": "text"}, + "field_016": {"type": "keyword"}, "field_017": {"type": "keyword"}, "field_018": {"type": "long"}, + "field_019": {"type": "keyword"}, "field_020": {"type": "text"}, "field_021": {"type": "long"}, + "field_022": {"type": "keyword"}, "field_023": {"type": "keyword"}, "field_024": {"type": "long"}, + "field_025": {"type": "text"}, "field_026": {"type": "keyword"}, "field_027": {"type": "long"}, + "field_028": {"type": "keyword"}, "field_029": {"type": "keyword"}, "field_030": {"type": "text"}, + "field_031": {"type": "keyword"}, "field_032": {"type": "keyword"}, "field_033": {"type": "long"}, + "field_034": {"type": "keyword"}, "field_035": {"type": "text"}, "field_036": {"type": "long"}, + "field_037": {"type": "keyword"}, "field_038": {"type": "keyword"}, "field_039": {"type": "long"}, + "field_040": {"type": "text"}, "field_041": {"type": "keyword"}, "field_042": {"type": "long"}, + "field_043": {"type": "keyword"}, "field_044": {"type": "keyword"}, "field_045": {"type": "text"}, + "field_046": {"type": "keyword"}, "field_047": {"type": "keyword"}, "field_048": {"type": "long"}, + "field_049": {"type": "keyword"}, "field_050": {"type": "text"}, "field_051": {"type": "long"}, + "field_052": {"type": "keyword"}, "field_053": {"type": "keyword"}, "field_054": {"type": "long"}, + "field_055": {"type": "text"}, "field_056": {"type": "keyword"}, "field_057": {"type": "long"}, + "field_058": {"type": "keyword"}, "field_059": {"type": "keyword"}, "field_060": {"type": "text"}, + "field_061": {"type": "keyword"}, "field_062": {"type": "keyword"}, "field_063": {"type": "long"}, + "field_064": {"type": "keyword"}, "field_065": {"type": "text"}, "field_066": {"type": "long"}, + "field_067": {"type": "keyword"}, "field_068": {"type": "keyword"}, "field_069": {"type": "long"}, + "field_070": {"type": "text"}, "field_071": {"type": "keyword"}, "field_072": {"type": "long"}, + "field_073": {"type": "keyword"}, "field_074": {"type": "keyword"}, "field_075": {"type": "text"}, + "field_076": {"type": "keyword"}, "field_077": {"type": "keyword"}, "field_078": {"type": "long"}, + "field_079": {"type": "keyword"}, "field_080": {"type": "text"}, "field_081": {"type": "long"}, + "field_082": {"type": "keyword"}, "field_083": {"type": "keyword"}, "field_084": {"type": "long"}, + "field_085": {"type": "text"}, "field_086": {"type": "keyword"}, "field_087": {"type": "long"}, + "field_088": {"type": "keyword"}, "field_089": {"type": "keyword"}, "field_090": {"type": "text"}, + "field_091": {"type": "keyword"}, "field_092": {"type": "keyword"}, "field_093": {"type": "long"}, + "field_094": {"type": "keyword"}, "field_095": {"type": "text"}, "field_096": {"type": "long"}, + "field_097": {"type": "keyword"}, "field_098": {"type": "keyword"}, "field_099": {"type": "long"}, + "field_100": {"type": "text"}, "field_101": {"type": "keyword"}, "field_102": {"type": "long"}, + "field_103": {"type": "keyword"}, "field_104": {"type": "keyword"}, "field_105": {"type": "text"}, + "field_106": {"type": "keyword"}, "field_107": {"type": "keyword"}, "field_108": {"type": "long"}, + "field_109": {"type": "keyword"}, "field_110": {"type": "text"}, "field_111": {"type": "long"}, + "field_112": {"type": "keyword"}, "field_113": {"type": "text"}, "field_114": {"type": "long"}, + "field_115": {"type": "text"}, "field_116": {"type": "keyword"}, "field_117": {"type": "long"}, + "field_118": {"type": "keyword"}, "field_119": {"type": "keyword"}, "field_120": {"type": "text"}, + "field_121": {"type": "keyword"}, "field_122": {"type": "keyword"}, "field_123": {"type": "long"}, + "field_124": {"type": "keyword"}, "field_125": {"type": "text"}, "field_126": {"type": "long"}, + "field_127": {"type": "keyword"}, "field_128": {"type": "keyword"}, "field_129": {"type": "long"}, + "field_130": {"type": "text"}, "field_131": {"type": "keyword"}, "field_132": {"type": "long"}, + "field_133": {"type": "keyword"}, "field_134": {"type": "keyword"}, "field_135": {"type": "text"}, + "field_136": {"type": "keyword"}, "field_137": {"type": "keyword"}, "field_138": {"type": "long"}, + "field_139": {"type": "keyword"}, "field_140": {"type": "text"}, "field_141": {"type": "long"}, + "field_142": {"type": "keyword"}, "field_143": {"type": "keyword"}, "field_144": {"type": "long"}, + "field_145": {"type": "text"}, "field_146": {"type": "keyword"}, "field_147": {"type": "long"}, + "field_148": {"type": "keyword"}, "field_149": {"type": "keyword"}, "field_150": {"type": "text"}, + "field_151": {"type": "keyword"}, "field_152": {"type": "keyword"}, "field_153": {"type": "long"}, + "field_154": {"type": "keyword"}, "field_155": {"type": "text"}, "field_156": {"type": "long"}, + "field_157": {"type": "keyword"}, "field_158": {"type": "keyword"}, "field_159": {"type": "long"}, + "field_160": {"type": "text"}, "field_161": {"type": "keyword"}, "field_162": {"type": "long"}, + "field_163": {"type": "keyword"}, "field_164": {"type": "keyword"}, "field_165": {"type": "text"}, + "field_166": {"type": "keyword"}, "field_167": {"type": "keyword"}, "field_168": {"type": "long"}, + "field_169": {"type": "keyword"}, "field_170": {"type": "text"}, "field_171": {"type": "long"}, + "field_172": {"type": "keyword"}, "field_173": {"type": "keyword"}, "field_174": {"type": "long"}, + "field_175": {"type": "text"}, "field_176": {"type": "keyword"}, "field_177": {"type": "long"}, + "field_178": {"type": "keyword"}, "field_179": {"type": "keyword"}, "field_180": {"type": "text"}, + "field_181": {"type": "keyword"}, "field_182": {"type": "keyword"}, "field_183": {"type": "long"}, + "field_184": {"type": "keyword"}, "field_185": {"type": "text"}, "field_186": {"type": "long"}, + "field_187": {"type": "keyword"}, "field_188": {"type": "keyword"}, "field_189": {"type": "long"}, + "field_190": {"type": "text"}, "field_191": {"type": "keyword"}, "field_192": {"type": "long"}, + "field_193": {"type": "keyword"}, "field_194": {"type": "keyword"}, "field_195": {"type": "text"}, + "field_196": {"type": "keyword"}, "field_197": {"type": "keyword"}, "field_198": {"type": "long"}, + "field_199": {"type": "keyword"}, "field_200": {"type": "text"}, "field_201": {"type": "long"}, + "field_202": {"type": "keyword"}, "field_203": {"type": "keyword"}, "field_204": {"type": "long"}, + "field_205": {"type": "text"}, "field_206": {"type": "keyword"}, "field_207": {"type": "long"}, + "field_208": {"type": "keyword"}, "field_209": {"type": "keyword"}, "field_210": {"type": "text"}, + "field_211": {"type": "keyword"}, "field_212": {"type": "keyword"}, "field_213": {"type": "long"}, + "field_214": {"type": "keyword"}, "field_215": {"type": "text"}, "field_216": {"type": "long"}, + "field_217": {"type": "keyword"}, "field_218": {"type": "keyword"}, "field_219": {"type": "long"}, + "field_220": {"type": "text"}, "field_221": {"type": "keyword"}, "field_222": {"type": "long"}, + "field_223": {"type": "keyword"}, "field_224": {"type": "keyword"}, "field_225": {"type": "text"}, + "field_226": {"type": "keyword"}, "field_227": {"type": "keyword"}, "field_228": {"type": "long"}, + "field_229": {"type": "keyword"}, "field_230": {"type": "text"}, "field_231": {"type": "long"}, + "field_232": {"type": "keyword"}, "field_233": {"type": "keyword"}, "field_234": {"type": "long"}, + "field_235": {"type": "text"}, "field_236": {"type": "keyword"}, "field_237": {"type": "long"}, + "field_238": {"type": "keyword"}, "field_239": {"type": "keyword"}, "field_240": {"type": "text"}, + "field_241": {"type": "keyword"}, "field_242": {"type": "keyword"}, "field_243": {"type": "long"}, + "field_244": {"type": "keyword"}, "field_245": {"type": "text"}, "field_246": {"type": "long"}, + "field_247": {"type": "keyword"}, "field_248": {"type": "keyword"}, "field_249": {"type": "long"}, + "field_250": {"type": "text"}, "field_251": {"type": "keyword"}, "field_252": {"type": "long"}, + "field_253": {"type": "keyword"}, "field_254": {"type": "keyword"}, "field_255": {"type": "text"}, + "field_256": {"type": "keyword"}, "field_257": {"type": "keyword"}, "field_258": {"type": "long"}, + "field_259": {"type": "keyword"}, "field_260": {"type": "text"}, "field_261": {"type": "long"}, + "field_262": {"type": "keyword"}, "field_263": {"type": "keyword"}, "field_264": {"type": "long"}, + "field_265": {"type": "text"}, "field_266": {"type": "keyword"}, "field_267": {"type": "long"}, + "field_268": {"type": "keyword"}, "field_269": {"type": "keyword"}, "field_270": {"type": "text"}, + "field_271": {"type": "keyword"}, "field_272": {"type": "keyword"}, "field_273": {"type": "long"}, + "field_274": {"type": "keyword"}, "field_275": {"type": "text"}, "field_276": {"type": "long"}, + "field_277": {"type": "keyword"}, "field_278": {"type": "keyword"}, "field_279": {"type": "long"}, + "field_280": {"type": "text"}, "field_281": {"type": "keyword"}, "field_282": {"type": "long"}, + "field_283": {"type": "keyword"}, "field_284": {"type": "keyword"}, "field_285": {"type": "text"}, + "field_286": {"type": "keyword"}, "field_287": {"type": "keyword"}, "field_288": {"type": "long"}, + "field_289": {"type": "keyword"}, "field_290": {"type": "text"}, "field_291": {"type": "long"}, + "field_292": {"type": "keyword"}, "field_293": {"type": "keyword"}, "field_294": {"type": "long"}, + "field_295": {"type": "text"}, "field_296": {"type": "keyword"}, "field_297": {"type": "long"}, + "field_298": {"type": "keyword"}, "field_299": {"type": "keyword"}, "field_300": {"type": "text"}, + "nested_001": {"type": "object", "properties": {"sub_001": {"type": "keyword"}, "sub_002": {"type": "keyword"}, "sub_003": {"type": "long"}, "sub_004": {"type": "text"}}}, + "nested_002": {"type": "object", "properties": {"sub_001": {"type": "keyword"}, "sub_002": {"type": "keyword"}, "sub_003": {"type": "long"}, "sub_004": {"type": "text"}}}, + "nested_003": {"type": "object", "properties": {"sub_001": {"type": "keyword"}, "sub_002": {"type": "keyword"}, "sub_003": {"type": "long"}, "sub_004": {"type": "text"}}}, + "nested_004": {"type": "object", "properties": {"sub_001": {"type": "keyword"}, "sub_002": {"type": "keyword"}, "sub_003": {"type": "long"}, "sub_004": {"type": "text"}}}, + "nested_005": {"type": "object", "properties": {"sub_001": {"type": "keyword"}, "sub_002": {"type": "keyword"}, "sub_003": {"type": "long"}, "sub_004": {"type": "text"}}}, + "nested_006": {"type": "object", "properties": {"sub_001": {"type": "keyword"}, "sub_002": {"type": "keyword"}, "sub_003": {"type": "long"}, "sub_004": {"type": "text"}}}, + "nested_007": {"type": "object", "properties": {"sub_001": {"type": "keyword"}, "sub_002": {"type": "keyword"}, "sub_003": {"type": "long"}, "sub_004": {"type": "text"}}}, + "nested_008": {"type": "object", "properties": {"sub_001": {"type": "keyword"}, "sub_002": {"type": "keyword"}, "sub_003": {"type": "long"}, "sub_004": {"type": "text"}}}, + "nested_009": {"type": "object", "properties": {"sub_001": {"type": "keyword"}, "sub_002": {"type": "keyword"}, "sub_003": {"type": "long"}, "sub_004": {"type": "text"}}}, + "nested_010": {"type": "object", "properties": {"sub_001": {"type": "keyword"}, "sub_002": {"type": "keyword"}, "sub_003": {"type": "long"}, "sub_004": {"type": "text"}}} + } + } + }' + + - do: + bulk: + index: test_wide_schema + refresh: true + body: + - '{"index":{}}' + - '{"activity_name":"login","start_time":"2025-01-14T10:00:00Z","end_time":"2025-01-14T10:05:30Z"}' + - '{"index":{}}' + - '{"activity_name":"logout","start_time":"2025-01-14T10:10:00Z","end_time":"2025-01-14T10:10:15Z"}' + - '{"index":{}}' + - '{"activity_name":"file_access","start_time":"2025-01-14T10:15:00Z","end_time":"2025-01-14T10:20:45Z"}' + - '{"index":{}}' + - '{"activity_name":"api_call","start_time":"2025-01-14T10:25:00Z","end_time":"2025-01-14T10:25:02Z"}' + - '{"index":{}}' + - '{"activity_name":"login","start_time":"2025-01-14T11:00:00Z","end_time":"2025-01-14T11:03:20Z"}' + - '{"index":{}}' + - '{"activity_name":"logout","start_time":"2025-01-14T11:05:00Z","end_time":"2025-01-14T11:05:10Z"}' + - '{"index":{}}' + - '{"activity_name":"file_access","start_time":"2025-01-14T11:10:00Z","end_time":"2025-01-14T11:18:30Z"}' + - '{"index":{}}' + - '{"activity_name":"api_call","start_time":"2025-01-14T11:20:00Z","end_time":"2025-01-14T11:20:01Z"}' + - '{"index":{}}' + - '{"activity_name":"login","start_time":"2025-01-14T12:00:00Z","end_time":"2025-01-14T12:04:15Z"}' + - '{"index":{}}' + - '{"activity_name":"logout","start_time":"2025-01-14T12:10:00Z","end_time":"2025-01-14T12:10:12Z"}' + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"Query with few required fields on wide schema index should succeed": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_wide_schema | eval start_ts = TIMESTAMP(start_time), end_ts = TIMESTAMP(end_time), duration = TIMESTAMPDIFF(SECOND, start_ts, end_ts) | where duration > 0 | stats count() as cnt by activity_name | sort - cnt | head 5 + - match: { total: 4 } + +--- +"Bin command spans over wide schema should succeed": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_wide_schema | bin start_time span=10minute | stats count() as cnt by start_time + - match: { total: 8 } diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4550.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4550.yml new file mode 100644 index 00000000000..147ab899552 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4550.yml @@ -0,0 +1,101 @@ +setup: + - do: + indices.create: + index: test_data_2023 + body: + mappings: + properties: + "@timestamp": + type: date + "packets": + type: integer + - do: + bulk: + index: test_data_2023 + refresh: true + body: + - '{"index":{}}' + - '{"@timestamp":"2023-10-08T10:00:00.000Z","packets":10}' + - '{"index":{}}' + - '{"@timestamp":"2023-10-08T10:00:00.500Z","packets":15}' + - '{"index":{}}' + - '{"@timestamp":"2023-10-08T10:00:01.000Z","packets":20}' + - '{"index":{}}' + - '{"@timestamp":"2023-10-08T10:00:01.500Z","packets":25}' + - '{"index":{}}' + - '{"@timestamp":"2023-10-08T10:00:02.000Z","packets":30}' + +--- +"timechart with millisecond span": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_data_2023 | timechart span=500ms count() + + - match: { total: 5 } + - match: { "schema": [ { "name": "@timestamp", "type": "timestamp" }, { "name": "count", "type": "bigint" }] } + - match: {"datarows": [["2023-10-08 10:00:00", 1], ["2023-10-08 10:00:00.5", 1], ["2023-10-08 10:00:01", 1], ["2023-10-08 10:00:01.5", 1], ["2023-10-08 10:00:02", 1]]} + +--- +"timechart with millisecond span and per_second function": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_data_2023 | timechart span=1000ms per_second(packets) + + - match: { total: 3 } + - match: { "schema": [ { "name": "@timestamp", "type": "timestamp" }, { "name": "per_second(packets)", "type": "double" }] } + - match: {"datarows": [["2023-10-08 10:00:00", 25.0], ["2023-10-08 10:00:01", 45.0], ["2023-10-08 10:00:02", 30.0]]} + +--- +"timechart with milliseconds": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_data_2023 | timechart span=250milliseconds count() + + - match: { total: 5 } + - match: { "schema": [ { "name": "@timestamp", "type": "timestamp" }, { "name": "count", "type": "bigint" }] } + +--- +"timechart with second span for comparison": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_data_2023 | timechart span=1s count() + + - match: { total: 3 } + - match: { "schema": [ { "name": "@timestamp", "type": "timestamp" }, { "name": "count", "type": "bigint" }] } + - match: {"datarows": [["2023-10-08 10:00:00", 2], ["2023-10-08 10:00:01", 2], ["2023-10-08 10:00:02", 1]]} diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4559.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4559.yml new file mode 100644 index 00000000000..397c913ac16 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4559.yml @@ -0,0 +1,71 @@ +setup: + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + aws: + properties: + cloudtrail: + properties: + event_name: + type: alias + path: api.operation + user_identity: + type: text + api: + properties: + operation: + type: keyword + + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"Handle nested alias field referring to the outer context": + - skip: + features: + - headers + - allowed_warnings + - do: + bulk: + index: test + refresh: true + body: + - '{"index": {}}' + - '{ + "aws": { + "cloudtrail": { + "user_identity": "test-user" + } + }, + "api": { + "operation": "CreateBucket" + } + }' + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: 'source=test | fields aws.cloudtrail.event_name' + - match: {"total": 1} + - match: { "schema": [ { "name": "aws.cloudtrail.event_name", "type": "string" } ] } + - match: { "datarows": [ [ "CreateBucket" ] ] } diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4563_4664.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4563_4664.yml new file mode 100644 index 00000000000..b5aa11876bb --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4563_4664.yml @@ -0,0 +1,69 @@ +setup: + - do: + indices.create: + index: test + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + + - do: + bulk: + index: test + refresh: true + body: + - '{"index": {}}' + - '{"status":"200","service":"api","value":100,"time":"2025-01-01T00:00:00Z"}' + - '{"index": {}}' + - '{"status":"500","service":"web","value":200,"time":"2025-01-02T00:00:00Z"}' + - '{"index": {}}' + - '{"status":"200","service":"db","value":150,"time":"2025-01-03T00:00:00Z"}' + - '{"index": {}}' + - '{"status":"404","service":"api","value":50,"time":"2025-01-03T00:01:00Z"}' + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"4563: Test rename then dedup": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test | rename status as http_status | dedup http_status | fields http_status + + - match: { total: 3 } + - match: { schema: [{"name": "http_status", "type": "string"}] } + - match: { datarows: [["200"], ["500"], ["404"]] } + +--- +"4664: Test rename then filter": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test | rename status as http_status | where http_status = '404' | fields http_status + + - match: { total: 1 } + - match: { schema: [{"name": "http_status", "type": "string"}] } + - match: { datarows: [["404"]] } diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4575.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4575.yml new file mode 100644 index 00000000000..81df3d23927 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4575.yml @@ -0,0 +1,65 @@ +setup: + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + a: + properties: + b: + properties: + c: + type: keyword + d: + properties: + e: + properties: + f: + type: keyword + num: + type: integer + + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"Access struct fields after join": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + bulk: + index: test + refresh: true + body: + - '{"index": {}}' + - '{"a": {"b": {"c": "/a/b/c"} }, "d": {"e": {"f": "/d/e/f"} }, "num": 10 }' + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: 'source=test | eval N=d.e.f | join type=inner left=l, right=r ON l.N = r.F [ source=test | rename d.e.f as F | fields F ] | fields num, N, a.b.c, d.e.f' + - match: {"total": 1} + - match: { schema: [ { "name": "num", "type": "int" }, { "name": "N", "type": "string" }, { "name": "a.b.c", "type": "string" }, { "name": "d.e.f", "type": "string" } ] } + - match: { datarows: [ [ 10, "/d/e/f", "/a/b/c", "/d/e/f"] ] } diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4580.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4580.yml new file mode 100644 index 00000000000..304c810ad83 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4580.yml @@ -0,0 +1,106 @@ +setup: + - do: + indices.create: + index: time_test + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "time_test"}}' + - '{"category":"A","value":1000,"@timestamp":"2024-01-01T00:00:00Z"}' + - '{"index": {"_index": "time_test"}}' + - '{"category":"B","value":2000,"@timestamp":"2024-01-01T00:05:00Z"}' + - '{"index": {"_index": "time_test"}}' + - '{"category":"A","value":1500,"@timestamp":"2024-01-01T00:10:00Z"}' + - '{"index": {"_index": "time_test"}}' + - '{"category":"C","value":3000,"@timestamp":"2024-01-01T00:20:00Z"}' + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"Test span aggregation with field name collision - basic case": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=time_test | stats count() by span(value, 1000) as value + + - match: { total: 3 } + - match: { schema: [{"name": "count()", "type": "bigint"}, {"name": "value", "type": "bigint"}] } + - match: { datarows: [[2, 1000], [1, 2000], [1, 3000]] } + +--- +"Test span aggregation with field name collision - multiple aggregations": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=time_test | stats count(), avg(value) by span(value, 1000) as value + + - match: { total: 3 } + - match: { schema: [{"name": "count()", "type": "bigint"}, {"name": "avg(value)", "type": "double"}, {"name": "value", "type": "bigint"}] } + - match: { datarows: [[2, 1250.0, 1000], [1, 2000.0, 2000], [1, 3000.0, 3000]] } + +--- +"Test span aggregation without name collision - multiple group-by": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=time_test | stats count() by span(@timestamp, 10min) as @timestamp, category, value + + - match: { total: 4 } + - match: { schema: [{"name": "count()", "type": "bigint"}, {"name": "@timestamp", "type": "timestamp"}, {"name": "category", "type": "string"}, {"name": "value", "type": "bigint"}] } + - match: { datarows: [[1, "2024-01-01 00:00:00", "A", 1000], [1, "2024-01-01 00:10:00", "A", 1500], [1, "2024-01-01 00:00:00", "B", 2000], [1, "2024-01-01 00:20:00", "C", 3000]] } + +--- +"Test span aggregation with duplicated group keys": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=time_test | stats count() by value, value, span(@timestamp, 10min) as @timestamp + + - match: { total: 4 } + - match: { schema: [{"name": "count()", "type": "bigint"}, {"name": "@timestamp", "type": "timestamp"}, {"name": "value", "type": "bigint"}, {"name": "value0", "type": "bigint"}] } + - match: { datarows: [[1, "2024-01-01 00:00:00", 1000, 1000], [1, "2024-01-01 00:10:00", 1500, 1500], [1, "2024-01-01 00:00:00", 2000, 2000], [1, "2024-01-01 00:20:00", 3000, 3000]] } diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4619.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4619.yml new file mode 100644 index 00000000000..cb1a9cc0e08 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4619.yml @@ -0,0 +1,83 @@ +setup: + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + log: + properties: + url: + properties: + message: + type: text + fields: + keyword: + type: keyword + ignore_above: 256 + time: + type: long + message_alias: + type: alias + path: log.url.message + time_alias: + type: alias + path: log.url.time + + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"dedup struct field name with dot": + - skip: + features: + - headers + - allowed_warnings + - do: + bulk: + index: test + refresh: true + body: + - '{"index": {}}' + - '{"log": {"url": {"message": "/e2e/h/zap", "time": 1} } }' + - '{"index": {}}' + - '{"log": {"url": {"message": "/e2e/h/zap", "time": 1} } }' + - '{"index": {}}' + - '{"log": {"url": {"message": "/e2e/h/zap", "time": 2} } }' + - '{"index": {}}' + - '{"log": {"url": {"message": "/e2e/h/zap", "time": 2} } }' + - '{"index": {}}' + - '{"log": {"url": {"message": "/e2e/h/aap", "time": 1} } }' + - '{"index": {}}' + - '{"log": {"url": {"message": "/e2e/h/aap", "time": 1} } }' + - '{"index": {}}' + - '{"log": {"url": {"time": 1} } }' + - '{"index": {}}' + - '{"log": {"url": {"time": 1} } }' + - '{"index": {}}' + - '{"log": {"url": {"time": 2} } }' + - '{"index": {}}' + - '{"log": {"url": {"time": 2} } }' + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: 'source=test | dedup log.url.time' + - match: {"total": 2} diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4705.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4705.yml new file mode 100644 index 00000000000..51d0c141d7a --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4705.yml @@ -0,0 +1,160 @@ +setup: + - do: + indices.create: + index: test + body: + mappings: + properties: + "dateV": + type: date + "intV": + type: integer + "boolV": + type: boolean + "stringV": + type: keyword + - do: + bulk: + index: test + refresh: true + body: + - '{"index":{}}' + - '{"dateV":"2023-10-08T10:00:00.000Z","intV":10,"boolV":true,"stringV":"hello"}' + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"String bucket parser should work in non-composite aggregate": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test | stats bucket_nullable=false count() by stringV + + - match: { total: 1 } + - match: { datarows: [[1, "hello"]] } + +--- +"Boolean bucket parser should work in non-composite aggregate": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test | stats bucket_nullable=false count() by boolV + + - match: { total: 1 } + - match: { datarows: [[1, true]] } + +--- +"Integer bucket parser should work in non-composite aggregate": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test | stats bucket_nullable=false count() by intV + + - match: { total: 1 } + - match: { datarows: [[1, 10]] } + +--- +"Date bucket parser should work in non-composite aggregate": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test | stats bucket_nullable=false count() by dateV + + - match: { total: 1 } + - match: { datarows: [[1, "2023-10-08 10:00:00"]] } + +--- +"Data histogram bucket parser should work in non-composite aggregate": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test | stats bucket_nullable=false count() by span(dateV, 1d) + + - match: { total: 1 } + - match: { datarows: [[1, "2023-10-08 00:00:00"]] } + +--- +"Histogram bucket parser should work in non-composite aggregate": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test | stats bucket_nullable=false count() by span(intV, 1) + + - match: { total: 1 } + - match: { datarows: [[1, 10]] } + +--- +"Multi-terms bucket parser should work in non-composite aggregate": + - skip: + features: + - headers + - allowed_warnings + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test | stats bucket_nullable=false count() by stringV, intV + + - match: { total: 1 } + - match: { datarows: [[1, "hello", 10]] } diff --git a/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 b/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 index 4c37be2f318..b7dc4b7286d 100644 --- a/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 +++ b/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 @@ -13,12 +13,9 @@ options { caseInsensitive = true; } SEARCH: 'SEARCH'; DESCRIBE: 'DESCRIBE'; SHOW: 'SHOW'; -EXPLAIN: 'EXPLAIN'; FROM: 'FROM'; WHERE: 'WHERE'; FIELDS: 'FIELDS'; -FIELD: 'FIELD'; -TABLE: 'TABLE'; // Alias for FIELDS command RENAME: 'RENAME'; STATS: 'STATS'; EVENTSTATS: 'EVENTSTATS'; @@ -26,14 +23,13 @@ DEDUP: 'DEDUP'; SORT: 'SORT'; EVAL: 'EVAL'; HEAD: 'HEAD'; -BIN: 'BIN'; +TOP_APPROX: 'TOP_APPROX'; TOP: 'TOP'; +RARE_APPROX: 'RARE_APPROX'; RARE: 'RARE'; PARSE: 'PARSE'; -SPATH: 'SPATH'; +METHOD: 'METHOD'; REGEX: 'REGEX'; -REX: 'REX'; -SED: 'SED'; PUNCT: 'PUNCT'; GROK: 'GROK'; PATTERN: 'PATTERN'; @@ -43,22 +39,10 @@ KMEANS: 'KMEANS'; AD: 'AD'; ML: 'ML'; FILLNULL: 'FILLNULL'; +EXPAND: 'EXPAND'; FLATTEN: 'FLATTEN'; TRENDLINE: 'TRENDLINE'; -TIMECHART: 'TIMECHART'; APPENDCOL: 'APPENDCOL'; -EXPAND: 'EXPAND'; -SIMPLE_PATTERN: 'SIMPLE_PATTERN'; -BRAIN: 'BRAIN'; -VARIABLE_COUNT_THRESHOLD: 'VARIABLE_COUNT_THRESHOLD'; -FREQUENCY_THRESHOLD_PERCENTAGE: 'FREQUENCY_THRESHOLD_PERCENTAGE'; -METHOD: 'METHOD'; -MAX_SAMPLE_COUNT: 'MAX_SAMPLE_COUNT'; -MAX_MATCH: 'MAX_MATCH'; -OFFSET_FIELD: 'OFFSET_FIELD'; -BUFFER_LIMIT: 'BUFFER_LIMIT'; -LABEL: 'LABEL'; -AGGREGATION: 'AGGREGATION'; //Native JOIN KEYWORDS JOIN: 'JOIN'; @@ -72,35 +56,52 @@ CROSS: 'CROSS'; LEFT_HINT: 'HINT.LEFT'; RIGHT_HINT: 'HINT.RIGHT'; +//CORRELATION KEYWORDS +CORRELATE: 'CORRELATE'; +SELF: 'SELF'; +EXACT: 'EXACT'; +APPROXIMATE: 'APPROXIMATE'; +SCOPE: 'SCOPE'; +MAPPING: 'MAPPING'; + +//EXPLAIN KEYWORDS +EXPLAIN: 'EXPLAIN'; +FORMATTED: 'FORMATTED'; +COST: 'COST'; +CODEGEN: 'CODEGEN'; +EXTENDED: 'EXTENDED'; +SIMPLE: 'SIMPLE'; + // COMMAND ASSIST KEYWORDS AS: 'AS'; BY: 'BY'; SOURCE: 'SOURCE'; INDEX: 'INDEX'; -A: 'A'; -ASC: 'ASC'; D: 'D'; DESC: 'DESC'; DATASOURCES: 'DATASOURCES'; USING: 'USING'; WITH: 'WITH'; -SIMPLE: 'SIMPLE'; -STANDARD: 'STANDARD'; -COST: 'COST'; -EXTENDED: 'EXTENDED'; -OVERRIDE: 'OVERRIDE'; -OVERWRITE: 'OVERWRITE'; // SORT FIELD KEYWORDS -// TODO #3180: Fix broken sort functionality +// TODO #963: Implement 'num', 'str', and 'ip' sort syntax AUTO: 'AUTO'; STR: 'STR'; +IP: 'IP'; NUM: 'NUM'; -// TRENDLINE KEYWORDS +// FIELDSUMMARY keywords +FIELDSUMMARY: 'FIELDSUMMARY'; +INCLUDEFIELDS: 'INCLUDEFIELDS'; +NULLS: 'NULLS'; + +//TRENDLINE KEYWORDS SMA: 'SMA'; WMA: 'WMA'; +// APPENDCOL options +OVERRIDE: 'OVERRIDE'; + // ARGUMENT KEYWORDS KEEPEMPTY: 'KEEPEMPTY'; CONSECUTIVE: 'CONSECUTIVE'; @@ -108,7 +109,6 @@ DEDUP_SPLITVALUES: 'DEDUP_SPLITVALUES'; PARTITIONS: 'PARTITIONS'; ALLNUM: 'ALLNUM'; DELIM: 'DELIM'; -BUCKET_NULLABLE: 'BUCKET_NULLABLE'; CENTROIDS: 'CENTROIDS'; ITERATIONS: 'ITERATIONS'; DISTANCE_TYPE: 'DISTANCE_TYPE'; @@ -124,13 +124,6 @@ TIME_ZONE: 'TIME_ZONE'; TRAINING_DATA_SIZE: 'TRAINING_DATA_SIZE'; ANOMALY_SCORE_THRESHOLD: 'ANOMALY_SCORE_THRESHOLD'; APPEND: 'APPEND'; -COUNTFIELD: 'COUNTFIELD'; -SHOWCOUNT: 'SHOWCOUNT'; -LIMIT: 'LIMIT'; -USEOTHER: 'USEOTHER'; -INPUT: 'INPUT'; -OUTPUT: 'OUTPUT'; -PATH: 'PATH'; // COMPARISON FUNCTION KEYWORDS CASE: 'CASE'; @@ -138,9 +131,6 @@ ELSE: 'ELSE'; IN: 'IN'; EXISTS: 'EXISTS'; -// Geo IP eval function -GEOIP: 'GEOIP'; - // LOGICAL KEYWORDS NOT: 'NOT'; OR: 'OR'; @@ -149,7 +139,6 @@ XOR: 'XOR'; TRUE: 'TRUE'; FALSE: 'FALSE'; REGEXP: 'REGEXP'; -REGEX_MATCH: 'REGEX_MATCH'; // DATETIME, INTERVAL AND UNIT KEYWORDS CONVERT_TZ: 'CONVERT_TZ'; @@ -197,14 +186,12 @@ LONG: 'LONG'; FLOAT: 'FLOAT'; STRING: 'STRING'; BOOLEAN: 'BOOLEAN'; -IP: 'IP'; // SPECIAL CHARACTERS AND OPERATORS PIPE: '|'; COMMA: ','; DOT: '.'; EQUAL: '='; -DOUBLE_EQUAL: '=='; GREATER: '>'; LESS: '<'; NOT_GREATER: '<' '='; @@ -221,8 +208,6 @@ LT_PRTHS: '('; RT_PRTHS: ')'; LT_SQR_PRTHS: '['; RT_SQR_PRTHS: ']'; -LT_CURLY: '{'; -RT_CURLY: '}'; SINGLE_QUOTE: '\''; DOUBLE_QUOTE: '"'; BACKTICK: '`'; @@ -255,12 +240,11 @@ VAR_SAMP: 'VAR_SAMP'; VAR_POP: 'VAR_POP'; STDDEV_SAMP: 'STDDEV_SAMP'; STDDEV_POP: 'STDDEV_POP'; -PERC: 'PERC'; PERCENTILE: 'PERCENTILE'; PERCENTILE_APPROX: 'PERCENTILE_APPROX'; -EARLIEST: 'EARLIEST'; -LATEST: 'LATEST'; TAKE: 'TAKE'; +FIRST: 'FIRST'; +LAST: 'LAST'; LIST: 'LIST'; VALUES: 'VALUES'; PER_DAY: 'PER_DAY'; @@ -272,22 +256,7 @@ SPARKLINE: 'SPARKLINE'; C: 'C'; DC: 'DC'; -// SCALAR WINDOW FUNCTIONS -ROW_NUMBER: 'ROW_NUMBER'; -RANK: 'RANK'; -DENSE_RANK: 'DENSE_RANK'; -PERCENT_RANK: 'PERCENT_RANK'; -CUME_DIST: 'CUME_DIST'; -FIRST: 'FIRST'; -LAST: 'LAST'; -NTH: 'NTH'; -NTILE: 'NTILE'; - // BASIC FUNCTIONS -PLUS_FUCTION: 'ADD'; -MINUS_FUCTION: 'SUBTRACT'; -STAR_FUNCTION: 'MULTIPLY'; -DIVIDE_FUNCTION: 'DIVIDE'; ABS: 'ABS'; CBRT: 'CBRT'; CEIL: 'CEIL'; @@ -296,13 +265,12 @@ CONV: 'CONV'; CRC32: 'CRC32'; E: 'E'; EXP: 'EXP'; -EXPM1: 'EXPM1'; FLOOR: 'FLOOR'; LN: 'LN'; LOG: 'LOG'; -LOG_WITH_BASE: ([0-9]+ ('.' [0-9]+)?)? ('LOG' | 'log') [0-9]+ ('.' [0-9]+)?; +LOG10: 'LOG10'; +LOG2: 'LOG2'; MOD: 'MOD'; -MODULUS: 'MODULUS'; PI: 'PI'; POSITION: 'POSITION'; POW: 'POW'; @@ -310,10 +278,9 @@ POWER: 'POWER'; RAND: 'RAND'; ROUND: 'ROUND'; SIGN: 'SIGN'; +SIGNUM: 'SIGNUM'; SQRT: 'SQRT'; TRUNCATE: 'TRUNCATE'; -RINT: 'RINT'; -SIGNUM: 'SIGNUM'; // TRIGONOMETRIC FUNCTIONS ACOS: 'ACOS'; @@ -321,12 +288,10 @@ ASIN: 'ASIN'; ATAN: 'ATAN'; ATAN2: 'ATAN2'; COS: 'COS'; -COSH: 'COSH'; COT: 'COT'; DEGREES: 'DEGREES'; RADIANS: 'RADIANS'; SIN: 'SIN'; -SINH: 'SINH'; TAN: 'TAN'; // CRYPTOGRAPHIC FUNCTIONS @@ -341,6 +306,7 @@ CURDATE: 'CURDATE'; CURRENT_DATE: 'CURRENT_DATE'; CURRENT_TIME: 'CURRENT_TIME'; CURRENT_TIMESTAMP: 'CURRENT_TIMESTAMP'; +CURRENT_TIMEZONE: 'CURRENT_TIMEZONE'; CURTIME: 'CURTIME'; DATE: 'DATE'; DATEDIFF: 'DATEDIFF'; @@ -353,6 +319,7 @@ DAYOFWEEK: 'DAYOFWEEK'; DAYOFYEAR: 'DAYOFYEAR'; DAY_OF_MONTH: 'DAY_OF_MONTH'; DAY_OF_WEEK: 'DAY_OF_WEEK'; +DURATION: 'DURATION'; EXTRACT: 'EXTRACT'; FROM_DAYS: 'FROM_DAYS'; FROM_UNIXTIME: 'FROM_UNIXTIME'; @@ -361,6 +328,7 @@ LAST_DAY: 'LAST_DAY'; LOCALTIME: 'LOCALTIME'; LOCALTIMESTAMP: 'LOCALTIMESTAMP'; MAKEDATE: 'MAKEDATE'; +MAKE_DATE: 'MAKE_DATE'; MAKETIME: 'MAKETIME'; MONTHNAME: 'MONTHNAME'; NOW: 'NOW'; @@ -387,6 +355,11 @@ UTC_TIMESTAMP: 'UTC_TIMESTAMP'; WEEKDAY: 'WEEKDAY'; YEARWEEK: 'YEARWEEK'; +// RELATIVE TIME FUNCTIONS +RELATIVE_TIMESTAMP: 'RELATIVE_TIMESTAMP'; +EARLIEST: 'EARLIEST'; +LATEST: 'LATEST'; + // TEXT FUNCTIONS SUBSTR: 'SUBSTR'; SUBSTRING: 'SUBSTRING'; @@ -408,45 +381,67 @@ REPLACE: 'REPLACE'; REVERSE: 'REVERSE'; CAST: 'CAST'; -// BOOL FUNCTIONS -LIKE: 'LIKE'; -ISNULL: 'ISNULL'; -ISNOTNULL: 'ISNOTNULL'; -CIDRMATCH: 'CIDRMATCH'; -BETWEEN: 'BETWEEN'; -ISPRESENT: 'ISPRESENT'; -ISEMPTY: 'ISEMPTY'; -ISBLANK: 'ISBLANK'; +// JSON TEXT FUNCTIONS +JSON: 'JSON'; +JSON_OBJECT: 'JSON_OBJECT'; +JSON_ARRAY: 'JSON_ARRAY'; +JSON_ARRAY_LENGTH: 'JSON_ARRAY_LENGTH'; +TO_JSON_STRING: 'TO_JSON_STRING'; +JSON_EXTRACT: 'JSON_EXTRACT'; +JSON_DELETE : 'JSON_DELETE'; +JSON_KEYS: 'JSON_KEYS'; +JSON_VALID: 'JSON_VALID'; +JSON_APPEND: 'JSON_APPEND'; +JSON_EXTEND : 'JSON_EXTEND'; +JSON_SET: 'JSON_SET'; +//JSON_ARRAY_ALL_MATCH: 'JSON_ARRAY_ALL_MATCH'; +//JSON_ARRAY_ANY_MATCH: 'JSON_ARRAY_ANY_MATCH'; +//JSON_ARRAY_FILTER: 'JSON_ARRAY_FILTER'; +//JSON_ARRAY_MAP: 'JSON_ARRAY_MAP'; +//JSON_ARRAY_REDUCE: 'JSON_ARRAY_REDUCE'; // COLLECTION FUNCTIONS ARRAY: 'ARRAY'; ARRAY_LENGTH: 'ARRAY_LENGTH'; -MVJOIN: 'MVJOIN'; + +// LAMBDA FUNCTIONS +//EXISTS: 'EXISTS'; FORALL: 'FORALL'; FILTER: 'FILTER'; TRANSFORM: 'TRANSFORM'; REDUCE: 'REDUCE'; -// JSON FUNCTIONS -JSON_VALID: 'JSON_VALID'; -JSON: 'JSON'; -JSON_OBJECT: 'JSON_OBJECT'; -JSON_ARRAY: 'JSON_ARRAY'; -JSON_ARRAY_LENGTH: 'JSON_ARRAY_LENGTH'; -JSON_EXTRACT: 'JSON_EXTRACT'; -JSON_KEYS: 'JSON_KEYS'; -JSON_SET: 'JSON_SET'; -JSON_DELETE: 'JSON_DELETE'; -JSON_APPEND: 'JSON_APPEND'; -JSON_EXTEND: 'JSON_EXTEND'; +// BOOL FUNCTIONS +LIKE: 'LIKE'; +ISNULL: 'ISNULL'; +ISNOTNULL: 'ISNOTNULL'; +BETWEEN: 'BETWEEN'; +CIDRMATCH: 'CIDRMATCH'; +ISPRESENT: 'ISPRESENT'; +ISEMPTY: 'ISEMPTY'; +ISBLANK: 'ISBLANK'; // FLOWCONTROL FUNCTIONS IFNULL: 'IFNULL'; NULLIF: 'NULLIF'; IF: 'IF'; TYPEOF: 'TYPEOF'; + +//OTHER CONDITIONAL EXPRESSIONS COALESCE: 'COALESCE'; +//GEOLOCATION FUNCTIONS +GEOIP: 'GEOIP'; + +//GEOLOCATION PROPERTIES +COUNTRY_ISO_CODE: 'COUNTRY_ISO_CODE'; +COUNTRY_NAME: 'COUNTRY_NAME'; +CONTINENT_NAME: 'CONTINENT_NAME'; +REGION_ISO_CODE: 'REGION_ISO_CODE'; +REGION_NAME: 'REGION_NAME'; +CITY_NAME: 'CITY_NAME'; +LOCATION: 'LOCATION'; + // RELEVANCE FUNCTIONS AND PARAMETERS MATCH: 'MATCH'; MATCH_PHRASE: 'MATCH_PHRASE'; @@ -490,11 +485,6 @@ ZERO_TERMS_QUERY: 'ZERO_TERMS_QUERY'; // SPAN KEYWORDS SPAN: 'SPAN'; -BINS: 'BINS'; -MINSPAN: 'MINSPAN'; -START: 'START'; -END: 'END'; -ALIGNTIME: 'ALIGNTIME'; MS: 'MS'; S: 'S'; M: 'M'; @@ -503,26 +493,6 @@ W: 'W'; Q: 'Q'; Y: 'Y'; -// Extended timescale units -SEC: 'SEC'; -SECS: 'SECS'; -SECONDS: 'SECONDS'; -MINS: 'MINS'; -MINUTES: 'MINUTES'; -HR: 'HR'; -HRS: 'HRS'; -HOURS: 'HOURS'; -DAYS: 'DAYS'; -MON: 'MON'; -MONTHS: 'MONTHS'; -US: 'US'; -CS: 'CS'; -DS: 'DS'; - - -// PERCENTILE SHORTCUT FUNCTIONS -// Must precede ID to avoid conflicts with identifier matching -PERCENTILE_SHORTCUT: PERC(INTEGER_LITERAL | DECIMAL_LITERAL) | 'P'(INTEGER_LITERAL | DECIMAL_LITERAL); // LITERALS AND VALUES //STRING_LITERAL: DQUOTA_STRING | SQUOTA_STRING | BQUOTA_STRING; @@ -530,10 +500,9 @@ ID: ID_LITERAL; CLUSTER: CLUSTER_PREFIX_LITERAL; INTEGER_LITERAL: DEC_DIGIT+; DECIMAL_LITERAL: (DEC_DIGIT+)? '.' DEC_DIGIT+; -FLOAT_LITERAL: (DEC_DIGIT+)? '.' DEC_DIGIT+ 'F'; -DOUBLE_LITERAL: (DEC_DIGIT+)? '.' DEC_DIGIT+ 'D'; fragment DATE_SUFFIX: ([\-.][*0-9]+)+; +fragment ID_LITERAL: [@*A-Z]+?[*A-Z_\-0-9]*; fragment CLUSTER_PREFIX_LITERAL: [*A-Z]+?[*A-Z_\-0-9]* COLON; ID_DATE_SUFFIX: CLUSTER_PREFIX_LITERAL? ID_LITERAL DATE_SUFFIX; DQUOTA_STRING: '"' ( '\\'. | '""' | ~('"'| '\\') )* '"'; @@ -541,10 +510,6 @@ SQUOTA_STRING: '\'' ('\\'. | '\'\'' | ~('\'' | '\\'))* '\'' BQUOTA_STRING: '`' ( '\\'. | '``' | ~('`'|'\\'))* '`'; fragment DEC_DIGIT: [0-9]; -// Identifiers cannot start with a single '_' since this an OpenSearch reserved -// metadata field. Two underscores (or more) is acceptable, such as '__field'. -fragment ID_LITERAL: ([@*A-Z_])+?[*A-Z_\-0-9]*; - LINE_COMMENT: '//' ('\\\n' | ~[\r\n])* '\r'? '\n'? -> channel(HIDDEN); BLOCK_COMMENT: '/*' .*? '*/' -> channel(HIDDEN); diff --git a/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 b/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 index d5cb4e3452b..cae57b53181 100644 --- a/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 +++ b/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 @@ -3,233 +3,168 @@ * SPDX-License-Identifier: Apache-2.0 */ - parser grammar OpenSearchPPLParser; options { tokenVocab = OpenSearchPPLLexer; } - root : pplStatement? EOF ; // statement pplStatement - : explainStatement - | queryStatement + : dmlStatement + ; + +dmlStatement + : (explainCommand PIPE)? queryStatement ; queryStatement : pplCommands (PIPE commands)* ; -explainStatement - : EXPLAIN (explainMode)? queryStatement - ; - -explainMode - : SIMPLE - | STANDARD - | COST - | EXTENDED - ; - subSearch : searchCommand (PIPE commands)* ; // commands pplCommands - : describeCommand - | showDataSourcesCommand - | searchCommand + : searchCommand + | describeCommand ; commands : whereCommand - | fieldsCommand - | tableCommand + | correlateCommand | joinCommand - | renameCommand + | fieldsCommand | statsCommand - | eventstatsCommand | dedupCommand | sortCommand - | evalCommand | headCommand - | binCommand | topCommand | rareCommand + | evalCommand | grokCommand | parseCommand - | spathCommand | patternsCommand | lookupCommand - | kmeansCommand - | adCommand - | mlCommand + | renameCommand | fillnullCommand + | fieldsummaryCommand + | flattenCommand + | expandCommand | trendlineCommand | appendcolCommand - | appendCommand - | expandCommand - | flattenCommand - | reverseCommand - | regexCommand - | timechartCommand - | rexCommand ; commandName : SEARCH | DESCRIBE | SHOW + | AD + | ML + | KMEANS | WHERE - | FIELDS - | TABLE + | CORRELATE | JOIN - | RENAME + | FIELDS | STATS | EVENTSTATS | DEDUP + | EXPLAIN | SORT - | EVAL | HEAD - | BIN | TOP + | TOP_APPROX | RARE + | RARE_APPROX + | EVAL | GROK | PARSE | PATTERNS | LOOKUP - | KMEANS - | AD - | ML - | FILLNULL + | RENAME | EXPAND + | FILLNULL + | FIELDSUMMARY | FLATTEN | TRENDLINE - | TIMECHART - | EXPLAIN - | REVERSE - | REGEX - | APPEND - | REX + | APPENDCOL ; searchCommand - : (SEARCH)? (searchExpression)* fromClause (searchExpression)* # searchFrom - ; - -searchExpression - : LT_PRTHS searchExpression RT_PRTHS # groupedExpression - | NOT searchExpression # notExpression - | searchExpression OR searchExpression # orExpression - | searchExpression AND searchExpression # andExpression - | searchTerm # termExpression - ; - -searchTerm - : searchFieldComparison # searchComparisonTerm - | searchFieldInList # searchInListTerm - | searchLiteral # searchLiteralTerm - ; - -// Unified search literal for both free text and field comparisons -searchLiteral - : numericLiteral - | booleanLiteral - | ID - | stringLiteral - | searchableKeyWord + : (SEARCH)? fromClause # searchFrom + | (SEARCH)? fromClause logicalExpression # searchFromFilter + | (SEARCH)? logicalExpression fromClause # searchFilterFrom ; -searchFieldComparison - : fieldExpression searchComparisonOperator searchLiteral # searchFieldCompare - ; - -searchFieldInList - : fieldExpression IN LT_PRTHS searchLiteralList RT_PRTHS # searchFieldInValues - ; - -searchLiteralList - : searchLiteral (COMMA searchLiteral)* # searchLiterals - ; - -searchComparisonOperator - : EQUAL # equals - | NOT_EQUAL # notEquals - | LESS # lessThan - | NOT_GREATER # lessOrEqual - | GREATER # greaterThan - | NOT_LESS # greaterOrEqual - ; - - -describeCommand - : DESCRIBE tableSourceClause +fieldsummaryCommand + : FIELDSUMMARY (fieldsummaryParameter)* ; -showDataSourcesCommand - : SHOW DATASOURCES +fieldsummaryParameter + : INCLUDEFIELDS EQUAL fieldList # fieldsummaryIncludeFields + | NULLS EQUAL booleanLiteral # fieldsummaryNulls ; -whereCommand - : WHERE logicalExpression - ; - -fieldsCommand - : FIELDS fieldsCommandBody +describeCommand + : DESCRIBE tableSourceClause ; -// Table command - alias for fields command -tableCommand - : TABLE fieldsCommandBody - ; +explainCommand + : EXPLAIN explainMode + ; -fieldsCommandBody - : (PLUS | MINUS)? wcFieldList - ; +explainMode + : FORMATTED + | COST + | CODEGEN + | EXTENDED + | SIMPLE + ; -// Wildcard field list supporting both comma-separated and space-separated fields -wcFieldList - : selectFieldExpression (COMMA? selectFieldExpression)* - ; +showDataSourcesCommand + : SHOW DATASOURCES + ; -renameCommand - : RENAME renameClasue (COMMA? renameClasue)* - ; +whereCommand + : WHERE logicalExpression + ; -statsCommand - : STATS statsArgs statsAggTerm (COMMA statsAggTerm)* (statsByClause)? (dedupSplitArg)? - ; +correlateCommand + : CORRELATE correlationType FIELDS LT_PRTHS fieldList RT_PRTHS (scopeClause)? mappingList + ; -statsArgs - : (partitionsArg | allnumArg | delimArg | bucketNullableArg)* - ; +correlationType + : SELF + | EXACT + | APPROXIMATE + ; -partitionsArg - : PARTITIONS EQUAL partitions = integerLiteral - ; +scopeClause + : SCOPE LT_PRTHS fieldExpression COMMA value = literalValue (unit = timespanUnit)? RT_PRTHS + ; -allnumArg - : ALLNUM EQUAL allnum = booleanLiteral - ; +mappingList + : MAPPING LT_PRTHS ( mappingClause (COMMA mappingClause)* ) RT_PRTHS + ; -delimArg - : DELIM EQUAL delim = stringLiteral - ; +mappingClause + : left = qualifiedName comparisonOperator right = qualifiedName # mappingCompareExpr + ; -bucketNullableArg - : BUCKET_NULLABLE EQUAL bucket_nullable = booleanLiteral +fieldsCommand + : FIELDS (PLUS | MINUS)? fieldList ; -dedupSplitArg - : DEDUP_SPLITVALUES EQUAL dedupsplit = booleanLiteral +renameCommand + : RENAME renameClasue (COMMA renameClasue)* ; -eventstatsCommand - : EVENTSTATS eventstatsAggTerm (COMMA eventstatsAggTerm)* (statsByClause)? +statsCommand + : (STATS | EVENTSTATS) (PARTITIONS EQUAL partitions = integerLiteral)? (ALLNUM EQUAL allnum = booleanLiteral)? (DELIM EQUAL delim = stringLiteral)? statsAggTerm (COMMA statsAggTerm)* (statsByClause)? (DEDUP_SPLITVALUES EQUAL dedupsplit = booleanLiteral)? ; dedupCommand @@ -237,30 +172,7 @@ dedupCommand ; sortCommand - : SORT (count = integerLiteral)? sortbyClause (ASC | A | DESC | D)? - ; - -reverseCommand - : REVERSE - ; - -timechartCommand - : TIMECHART timechartParameter* statsFunction (BY fieldExpression)? - ; - -timechartParameter - : (spanClause | SPAN EQUAL spanLiteral) - | timechartArg - ; - -timechartArg - : LIMIT EQUAL integerLiteral - | USEOTHER EQUAL (booleanLiteral | ident) - ; - -spanLiteral - : integerLiteral timespanUnit - | stringLiteral + : SORT sortbyClause ; evalCommand @@ -271,42 +183,12 @@ headCommand : HEAD (number = integerLiteral)? (FROM from = integerLiteral)? ; -binCommand - : BIN fieldExpression binOption* (AS alias = qualifiedName)? - ; - -binOption - : SPAN EQUAL span = spanValue - | BINS EQUAL bins = integerLiteral - | MINSPAN EQUAL minspan = literalValue (minspanUnit = timespanUnit)? - | ALIGNTIME EQUAL aligntime = aligntimeValue - | START EQUAL start = numericLiteral - | END EQUAL end = numericLiteral - ; - -aligntimeValue - : EARLIEST - | LATEST - | literalValue - ; - -spanValue - : literalValue (timespanUnit)? # numericSpanValue - | logSpanValue # logBasedSpanValue - | ident timespanUnit # extendedTimeSpanValue - | ident # identifierSpanValue - ; - -logSpanValue - : LOG_WITH_BASE # logWithBaseSpan - ; - topCommand - : TOP (number = integerLiteral)? (COUNTFIELD EQUAL countfield = stringLiteral)? (SHOWCOUNT EQUAL showcount = booleanLiteral)? fieldList (byClause)? + : (TOP | TOP_APPROX) (number = integerLiteral)? fieldList (byClause)? ; rareCommand - : RARE (number = integerLiteral)? (COUNTFIELD EQUAL countfield = stringLiteral)? (SHOWCOUNT EQUAL showcount = booleanLiteral)? fieldList (byClause)? + : (RARE | RARE_APPROX) (number = integerLiteral)? fieldList (byClause)? ; grokCommand @@ -317,73 +199,20 @@ parseCommand : PARSE (source_field = expression) (pattern = stringLiteral) ; -spathCommand - : SPATH spathParameter* - ; - -spathParameter - : (INPUT EQUAL input = expression) - | (OUTPUT EQUAL output = expression) - | ((PATH EQUAL)? path = indexablePath) - ; - -indexablePath - : pathElement (DOT pathElement)* - ; - -pathElement - : ident pathArrayAccess? +patternsCommand + : PATTERNS (patternsParameter)* (source_field = expression) ; -pathArrayAccess - : LT_CURLY (INTEGER_LITERAL)? RT_CURLY +patternsParameter + : (NEW_FIELD EQUAL new_field = stringLiteral) + | (PATTERN EQUAL pattern = stringLiteral) ; -regexCommand - : REGEX regexExpr - ; - -regexExpr - : field=qualifiedName operator=(EQUAL | NOT_EQUAL) pattern=stringLiteral - ; - -rexCommand - : REX rexExpr - ; -rexExpr - : FIELD EQUAL field=qualifiedName (rexOption)* pattern=stringLiteral (rexOption)* - ; - -rexOption - : MAX_MATCH EQUAL maxMatch=integerLiteral - | MODE EQUAL (EXTRACT | SED) - | OFFSET_FIELD EQUAL offsetField=qualifiedName - ; patternsMethod : PUNCT | REGEX ; -patternsCommand - : PATTERNS (source_field = expression) (statsByClause)? (METHOD EQUAL method = patternMethod)? (MODE EQUAL pattern_mode = patternMode)? (MAX_SAMPLE_COUNT EQUAL max_sample_count = integerLiteral)? (BUFFER_LIMIT EQUAL buffer_limit = integerLiteral)? (NEW_FIELD EQUAL new_field = stringLiteral)? (patternsParameter)* - ; - -patternsParameter - : (PATTERN EQUAL pattern = stringLiteral) - | (VARIABLE_COUNT_THRESHOLD EQUAL variable_count_threshold = integerLiteral) - | (FREQUENCY_THRESHOLD_PERCENTAGE EQUAL frequency_threshold_percentage = decimalLiteral) - ; - -patternMethod - : SIMPLE_PATTERN - | BRAIN - ; - -patternMode - : LABEL - | AGGREGATION - ; - // lookup lookupCommand : LOOKUP tableSource lookupMappingList ((APPEND | REPLACE) outputCandidateList)? @@ -406,28 +235,36 @@ lookupPair ; fillnullCommand - : FILLNULL fillNullWith - | FILLNULL fillNullUsing + : FILLNULL (fillNullWithTheSameValue + | fillNullWithFieldVariousValues) ; -fillNullWith - : WITH replacement = valueExpression (IN fieldList)? +fillNullWithTheSameValue + : WITH nullReplacement = valueExpression IN nullableFieldList = fieldList ; -fillNullUsing - : USING replacementPair (COMMA replacementPair)* +fillNullWithFieldVariousValues + : USING nullableReplacementExpression (COMMA nullableReplacementExpression)* ; -replacementPair - : fieldExpression EQUAL replacement = valueExpression +nullableReplacementExpression + : nullableField = fieldExpression EQUAL nullableReplacement = valueExpression ; +expandCommand + : EXPAND fieldExpression (AS alias = qualifiedName)? + ; + +flattenCommand + : FLATTEN fieldExpression (AS alias = identifierSeq)? + ; + trendlineCommand : TRENDLINE (SORT sortField)? trendlineClause (trendlineClause)* ; trendlineClause - : trendlineType LT_PRTHS numberOfDataPoints = integerLiteral COMMA field = fieldExpression RT_PRTHS (AS alias = qualifiedName)? + : trendlineType LT_PRTHS numberOfDataPoints = INTEGER_LITERAL COMMA field = fieldExpression RT_PRTHS (AS alias = qualifiedName)? ; trendlineType @@ -435,22 +272,10 @@ trendlineType | WMA ; -expandCommand - : EXPAND fieldExpression (AS alias = qualifiedName)? - ; - -flattenCommand - : FLATTEN fieldExpression (AS aliases = identifierSeq)? - ; - appendcolCommand : APPENDCOL (OVERRIDE EQUAL override = booleanLiteral)? LT_SQR_PRTHS commands (PIPE commands)* RT_SQR_PRTHS ; -appendCommand - : APPEND LT_SQR_PRTHS searchCommand? (PIPE commands)* RT_SQR_PRTHS - ; - kmeansCommand : KMEANS (kmeansParameter)* ; @@ -492,10 +317,6 @@ mlArg fromClause : SOURCE EQUAL tableOrSubqueryClause | INDEX EQUAL tableOrSubqueryClause - | SOURCE EQUAL tableFunction - | INDEX EQUAL tableFunction - | SOURCE EQUAL dynamicSourceClause - | INDEX EQUAL dynamicSourceClause ; tableOrSubqueryClause @@ -503,64 +324,36 @@ tableOrSubqueryClause | tableSourceClause ; +// One tableSourceClause will generate one Relation node with/without one alias +// even if the relation contains more than one table sources. +// These table sources in one relation will be readed one by one in OpenSearch. +// But it may have different behaivours in different execution backends. +// For example, a Spark UnresovledRelation node only accepts one data source. tableSourceClause : tableSource (COMMA tableSource)* (AS alias = qualifiedName)? ; -dynamicSourceClause - : LT_SQR_PRTHS sourceReferences (COMMA sourceFilterArgs)? RT_SQR_PRTHS - ; - -sourceReferences - : sourceReference (COMMA sourceReference)* - ; - -sourceReference - : (CLUSTER)? wcQualifiedName - ; - -sourceFilterArgs - : sourceFilterArg (COMMA sourceFilterArg)* - ; - -sourceFilterArg - : ident EQUAL literalValue - | ident IN valueList - ; - // join joinCommand - : JOIN (joinOption)* (fieldList)? right = tableOrSubqueryClause - | sqlLikeJoinType? JOIN (joinOption)* sideAlias joinHintList? joinCriteria right = tableOrSubqueryClause + : (joinType) JOIN sideAlias joinHintList? joinCriteria? right = tableOrSubqueryClause ; -sqlLikeJoinType - : INNER +joinType + : INNER? | CROSS - | (LEFT OUTER? | OUTER) + | LEFT OUTER? | RIGHT OUTER? | FULL OUTER? | LEFT? SEMI | LEFT? ANTI ; -joinType - : INNER - | CROSS - | OUTER - | LEFT - | RIGHT - | FULL - | SEMI - | ANTI - ; - sideAlias : (LEFT EQUAL leftAlias = qualifiedName)? COMMA? (RIGHT EQUAL rightAlias = qualifiedName)? ; joinCriteria - : (ON | WHERE) logicalExpression + : ON logicalExpression ; joinHintList @@ -572,14 +365,8 @@ hintPair | rightHintKey = RIGHT_HINT DOT ID EQUAL rightHintValue = ident #rightHint ; -joinOption - : OVERWRITE EQUAL booleanLiteral # overwriteOption - | TYPE EQUAL joinType # typeOption - | MAX EQUAL integerLiteral # maxOption - ; - renameClasue - : orignalField = renameFieldExpression AS renamedField = renameFieldExpression + : orignalField = wcFieldExpression AS renamedField = wcFieldExpression ; byClause @@ -590,7 +377,6 @@ statsByClause : BY fieldList | BY bySpanClause | BY bySpanClause COMMA fieldList - | BY fieldList COMMA bySpanClause ; bySpanClause @@ -606,34 +392,12 @@ sortbyClause ; evalClause - : fieldExpression EQUAL logicalExpression + : fieldExpression EQUAL expression + | geoipCommand ; -eventstatsAggTerm - : windowFunction (AS alias = wcFieldExpression)? - ; - -windowFunction - : windowFunctionName LT_PRTHS functionArgs RT_PRTHS - ; - -windowFunctionName - : statsFunctionName - | scalarWindowFunctionName - ; - -scalarWindowFunctionName - : ROW_NUMBER - | RANK - | DENSE_RANK - | PERCENT_RANK - | CUME_DIST - | FIRST - | LAST - | NTH - | NTILE - | DISTINCT_COUNT - | DC +geoipCommand + : fieldExpression EQUAL GEOIP LT_PRTHS ipAddress = functionArg (COMMA properties = geoIpPropertyList)? RT_PRTHS ; // aggregation terms @@ -643,13 +407,10 @@ statsAggTerm // aggregation functions statsFunction - : (COUNT | C) LT_PRTHS evalExpression RT_PRTHS # countEvalFunctionCall - | (COUNT | C) (LT_PRTHS RT_PRTHS)? # countAllFunctionCall - | PERCENTILE_SHORTCUT LT_PRTHS valueExpression RT_PRTHS # percentileShortcutFunctionCall - | (DISTINCT_COUNT | DC | DISTINCT_COUNT_APPROX) LT_PRTHS valueExpression RT_PRTHS # distinctCountFunctionCall - | takeAggFunction # takeAggFunctionCall - | percentileApproxFunction # percentileApproxFunctionCall - | statsFunctionName LT_PRTHS functionArgs RT_PRTHS # statsFunctionCall + : statsFunctionName LT_PRTHS valueExpression RT_PRTHS # statsFunctionCall + | COUNT LT_PRTHS RT_PRTHS # countAllFunctionCall + | (DISTINCT_COUNT | DC | DISTINCT_COUNT_APPROX) LT_PRTHS valueExpression RT_PRTHS # distinctCountFunctionCall + | percentileFunctionName = (PERCENTILE | PERCENTILE_APPROX) LT_PRTHS valueExpression COMMA percent = integerLiteral RT_PRTHS # percentileFunctionCall ; statsFunctionName @@ -658,89 +419,72 @@ statsFunctionName | SUM | MIN | MAX - | VAR_SAMP - | VAR_POP | STDDEV_SAMP | STDDEV_POP - | PERCENTILE - | PERCENTILE_APPROX - | MEDIAN - | LIST - | FIRST - | EARLIEST - | LATEST - | LAST ; -takeAggFunction - : TAKE LT_PRTHS fieldExpression (COMMA size = integerLiteral)? RT_PRTHS - ; - -percentileApproxFunction - : (PERCENTILE | PERCENTILE_APPROX) LT_PRTHS aggField = valueExpression - COMMA percent = numericLiteral (COMMA compression = numericLiteral)? RT_PRTHS +// expressions +expression + : logicalExpression + | valueExpression ; -numericLiteral - : integerLiteral - | decimalLiteral - | doubleLiteral - | floatLiteral - ; - -// predicates logicalExpression : NOT logicalExpression # logicalNot - | left = logicalExpression AND right = logicalExpression # logicalAnd - | left = logicalExpression XOR right = logicalExpression # logicalXor + | LT_PRTHS logicalExpression RT_PRTHS # parentheticLogicalExpr + | comparisonExpression # comparsion + | left = logicalExpression (AND)? right = logicalExpression # logicalAnd | left = logicalExpression OR right = logicalExpression # logicalOr - | expression # logicalExpr + | left = logicalExpression XOR right = logicalExpression # logicalXor + | booleanExpression # booleanExpr ; -expression - : valueExpression # valueExpr - | relevanceExpression # relevanceExpr - | left = expression comparisonOperator right = expression # compareExpr - | expression NOT? IN valueList # inExpr - | expression NOT? BETWEEN expression AND expression # between +comparisonExpression + : left = valueExpression comparisonOperator right = valueExpression # compareExpr + | valueExpression NOT? IN valueList # inExpr + | expr1 = functionArg NOT? BETWEEN expr2 = functionArg AND expr3 = functionArg # between ; -valueExpression - : left = valueExpression binaryOperator = (STAR | DIVIDE | MODULE) right = valueExpression # binaryArithmetic - | left = valueExpression binaryOperator = (PLUS | MINUS) right = valueExpression # binaryArithmetic - | literalValue # literalValueExpr - | functionCall # functionCallExpr - | lambda # lambdaExpr - | LT_SQR_PRTHS subSearch RT_SQR_PRTHS # scalarSubqueryExpr - | valueExpression NOT? IN LT_SQR_PRTHS subSearch RT_SQR_PRTHS # inSubqueryExpr - | LT_PRTHS valueExpression (COMMA valueExpression)* RT_PRTHS NOT? IN LT_SQR_PRTHS subSearch RT_SQR_PRTHS # inSubqueryExpr - | EXISTS LT_SQR_PRTHS subSearch RT_SQR_PRTHS # existsSubqueryExpr - | fieldExpression # fieldExpr - | LT_PRTHS logicalExpression RT_PRTHS # nestedValueExpr - ; - -evalExpression - : EVAL LT_PRTHS logicalExpression RT_PRTHS - ; +valueExpressionList + : valueExpression + | LT_PRTHS valueExpression (COMMA valueExpression)* RT_PRTHS + ; -functionCall +valueExpression + : left = valueExpression binaryOperator = (STAR | DIVIDE | MODULE) right = valueExpression # binaryArithmetic + | left = valueExpression binaryOperator = (PLUS | MINUS) right = valueExpression # binaryArithmetic + | primaryExpression # valueExpressionDefault + | positionFunction # positionFunctionCall + | caseFunction # caseExpr + | timestampFunction # timestampFunctionCall + | LT_PRTHS valueExpression RT_PRTHS # parentheticValueExpr + | LT_SQR_PRTHS subSearch RT_SQR_PRTHS # scalarSubqueryExpr + | ident ARROW expression # lambda + | LT_PRTHS ident (COMMA ident)+ RT_PRTHS ARROW expression # lambda + ; + +primaryExpression : evalFunctionCall + | fieldExpression + | literalValue | dataTypeFunctionCall - | positionFunctionCall - | caseFunctionCall - | timestampFunctionCall - | extractFunctionCall - | getFormatFunctionCall ; -positionFunctionCall +positionFunction : positionFunctionName LT_PRTHS functionArg IN functionArg RT_PRTHS ; -caseFunctionCall - : CASE LT_PRTHS logicalExpression COMMA valueExpression (COMMA logicalExpression COMMA valueExpression)* (ELSE valueExpression)? RT_PRTHS +booleanExpression + : booleanFunctionCall # booleanFunctionCallExpr + | valueExpressionList NOT? IN LT_SQR_PRTHS subSearch RT_SQR_PRTHS # inSubqueryExpr + | EXISTS LT_SQR_PRTHS subSearch RT_SQR_PRTHS # existsSubqueryExpr + | cidrMatchFunctionCall # cidrFunctionCallExpr ; + caseFunction + : CASE LT_PRTHS logicalExpression COMMA valueExpression (COMMA logicalExpression COMMA valueExpression)* (ELSE valueExpression)? RT_PRTHS + ; + relevanceExpression : singleFieldRelevanceFunction | multiFieldRelevanceFunction @@ -753,7 +497,7 @@ singleFieldRelevanceFunction // Field is a list of columns multiFieldRelevanceFunction - : multiFieldRelevanceFunctionName LT_PRTHS (LT_SQR_PRTHS field = relevanceFieldAndWeight (COMMA field = relevanceFieldAndWeight)* RT_SQR_PRTHS COMMA)? query = relevanceQuery (COMMA relevanceArg)* RT_PRTHS + : multiFieldRelevanceFunctionName LT_PRTHS LT_SQR_PRTHS field = relevanceFieldAndWeight (COMMA field = relevanceFieldAndWeight)* RT_SQR_PRTHS COMMA query = relevanceQuery (COMMA relevanceArg)* RT_PRTHS ; // tables @@ -763,12 +507,16 @@ tableSource ; tableFunction - : qualifiedName LT_PRTHS namedFunctionArgs RT_PRTHS + : qualifiedName LT_PRTHS functionArgs RT_PRTHS ; // fields fieldList - : fieldExpression ((COMMA)? fieldExpression)* + : fieldExpression (COMMA fieldExpression)* + ; + +wcFieldList + : wcFieldExpression (COMMA wcFieldExpression)* ; sortField @@ -777,6 +525,8 @@ sortField sortFieldExpression : fieldExpression + + // TODO #963: Implement 'num', 'str', and 'ip' sort syntax | AUTO LT_PRTHS fieldExpression RT_PRTHS | STR LT_PRTHS fieldExpression RT_PRTHS | IP LT_PRTHS fieldExpression RT_PRTHS @@ -791,16 +541,6 @@ wcFieldExpression : wcQualifiedName ; -selectFieldExpression - : wcQualifiedName - | STAR - ; - -renameFieldExpression - : wcQualifiedName - | STAR - ; - // functions evalFunctionCall : evalFunctionName LT_PRTHS functionArgs RT_PRTHS @@ -808,7 +548,16 @@ evalFunctionCall // cast function dataTypeFunctionCall - : CAST LT_PRTHS logicalExpression AS convertedDataType RT_PRTHS + : CAST LT_PRTHS expression AS convertedDataType RT_PRTHS + ; + +// boolean functions +booleanFunctionCall + : conditionFunctionBase LT_PRTHS functionArgs RT_PRTHS + ; + +cidrMatchFunctionCall + : CIDRMATCH LT_PRTHS ipAddress = functionArg COMMA cidrBlock = functionArg RT_PRTHS ; convertedDataType @@ -822,48 +571,28 @@ convertedDataType | typeName = FLOAT | typeName = STRING | typeName = BOOLEAN - | typeName = IP - | typeName = JSON ; evalFunctionName : mathematicalFunctionName | dateTimeFunctionName | textFunctionName - | conditionFunctionName - | flowControlFunctionName + | conditionFunctionBase | systemFunctionName | positionFunctionName + | coalesceFunctionName | cryptographicFunctionName | jsonFunctionName - | geoipFunctionName | collectionFunctionName + | lambdaFunctionName ; functionArgs : (functionArg (COMMA functionArg)*)? ; -namedFunctionArgs - : (namedFunctionArg (COMMA namedFunctionArg)*)? - ; - functionArg - : functionArgExpression - ; - -namedFunctionArg - : (ident EQUAL)? functionArgExpression - ; - -functionArgExpression - : lambda - | logicalExpression - ; - -lambda - : ident ARROW logicalExpression - | LT_PRTHS ident (COMMA ident)+ RT_PRTHS ARROW logicalExpression + : (ident EQUAL)? valueExpression ; relevanceArg @@ -915,8 +644,6 @@ relevanceFieldAndWeight relevanceFieldWeight : integerLiteral | decimalLiteral - | doubleLiteral - | floatLiteral ; relevanceField @@ -935,10 +662,6 @@ relevanceArgValue mathematicalFunctionName : ABS - | PLUS_FUCTION - | MINUS_FUCTION - | STAR_FUNCTION - | DIVIDE_FUNCTION | CBRT | CEIL | CEILING @@ -946,72 +669,37 @@ mathematicalFunctionName | CRC32 | E | EXP - | EXPM1 | FLOOR | LN | LOG - | LOG_WITH_BASE + | LOG10 + | LOG2 | MOD - | MODULUS | PI | POW | POWER | RAND | ROUND | SIGN + | SIGNUM | SQRT | TRUNCATE - | RINT - | SIGNUM - | SUM - | AVG | trigonometricFunctionName ; -geoipFunctionName - : GEOIP - ; - -collectionFunctionName - : ARRAY - | ARRAY_LENGTH - | MVJOIN - | FORALL - | EXISTS - | FILTER - | TRANSFORM - | REDUCE - ; - - trigonometricFunctionName : ACOS | ASIN | ATAN | ATAN2 | COS - | COSH | COT | DEGREES | RADIANS | SIN - | SINH | TAN ; -jsonFunctionName - : JSON - | JSON_OBJECT - | JSON_ARRAY - | JSON_ARRAY_LENGTH - | JSON_EXTRACT - | JSON_KEYS - | JSON_SET - | JSON_DELETE - | JSON_APPEND - | JSON_EXTEND - ; - cryptographicFunctionName : MD5 | SHA1 @@ -1026,6 +714,7 @@ dateTimeFunctionName | CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP + | CURRENT_TIMEZONE | CURTIME | DATE | DATEDIFF @@ -1049,6 +738,7 @@ dateTimeFunctionName | LOCALTIME | LOCALTIMESTAMP | MAKEDATE + | MAKE_DATE | MAKETIME | MICROSECOND | MINUTE @@ -1084,9 +774,16 @@ dateTimeFunctionName | WEEK_OF_YEAR | YEAR | YEARWEEK + | relativeTimeFunctionName ; -getFormatFunctionCall +relativeTimeFunctionName + : RELATIVE_TIMESTAMP + | EARLIEST + | LATEST + ; + +getFormatFunction : GET_FORMAT LT_PRTHS getFormatType COMMA functionArg RT_PRTHS ; @@ -1097,7 +794,7 @@ getFormatType | TIMESTAMP ; -extractFunctionCall +extractFunction : EXTRACT LT_PRTHS datetimePart FROM functionArg RT_PRTHS ; @@ -1132,7 +829,7 @@ datetimePart | complexDateTimePart ; -timestampFunctionCall +timestampFunction : timestampFunctionName LT_PRTHS simpleDateTimePart COMMA firstArg = functionArg COMMA secondArg = functionArg RT_PRTHS ; @@ -1142,26 +839,19 @@ timestampFunctionName ; // condition function return boolean value -conditionFunctionName +conditionFunctionBase : LIKE + | IF | ISNULL | ISNOTNULL - | CIDRMATCH - | REGEX_MATCH - | JSON_VALID + | IFNULL + | NULLIF | ISPRESENT - | ISEMPTY - | ISBLANK + | JSON_VALID | EARLIEST | LATEST - ; - -// flow control function return non-boolean value -flowControlFunctionName - : IF - | IFNULL - | NULLIF - | COALESCE + | ISEMPTY + | ISBLANK ; systemFunctionName @@ -1186,23 +876,75 @@ textFunctionName | LOCATE | REPLACE | REVERSE + | ISEMPTY + | ISBLANK + ; + +jsonFunctionName + : JSON + | JSON_OBJECT + | JSON_ARRAY + | JSON_ARRAY_LENGTH + | TO_JSON_STRING + | JSON_EXTRACT + | JSON_DELETE + | JSON_APPEND + | JSON_KEYS + | JSON_VALID + | JSON_EXTEND + | JSON_SET +// | JSON_ARRAY_ALL_MATCH +// | JSON_ARRAY_ANY_MATCH +// | JSON_ARRAY_FILTER +// | JSON_ARRAY_MAP +// | JSON_ARRAY_REDUCE ; +collectionFunctionName + : ARRAY + | ARRAY_LENGTH + ; + +lambdaFunctionName + : FORALL + | EXISTS + | FILTER + | TRANSFORM + | REDUCE + ; + positionFunctionName : POSITION ; +coalesceFunctionName + : COALESCE + ; + +geoIpPropertyList + : geoIpProperty (COMMA geoIpProperty)* + ; + +geoIpProperty + : COUNTRY_ISO_CODE + | COUNTRY_NAME + | CONTINENT_NAME + | REGION_ISO_CODE + | REGION_NAME + | CITY_NAME + | TIME_ZONE + | LOCATION + ; + // operators comparisonOperator : EQUAL - | DOUBLE_EQUAL | NOT_EQUAL | LESS | NOT_LESS | GREATER | NOT_GREATER | REGEXP - | LIKE ; singleFieldRelevanceFunctionName @@ -1220,14 +962,12 @@ multiFieldRelevanceFunctionName // literals and values literalValue - : intervalLiteral - | stringLiteral + : stringLiteral | integerLiteral | decimalLiteral - | doubleLiteral - | floatLiteral | booleanLiteral | datetimeLiteral //#datetime + | intervalLiteral ; intervalLiteral @@ -1247,14 +987,6 @@ decimalLiteral : (PLUS | MINUS)? DECIMAL_LITERAL ; -doubleLiteral - : (PLUS | MINUS)? DOUBLE_LITERAL - ; - -floatLiteral - : (PLUS | MINUS)? FLOAT_LITERAL - ; - booleanLiteral : TRUE | FALSE @@ -1320,20 +1052,6 @@ timespanUnit | MONTH | QUARTER | YEAR - | SEC - | SECS - | SECONDS - | MINS - | MINUTES - | HR - | HRS - | HOURS - | DAYS - | MON - | MONTHS - | US - | CS - | DS ; valueList @@ -1344,6 +1062,11 @@ qualifiedName : ident (DOT ident)* # identsAsQualifiedName ; +identifierSeq + : qualifiedName (COMMA qualifiedName)* # identsAsQualifiedNameSeq + | LT_PRTHS qualifiedName (COMMA qualifiedName)* RT_PRTHS # identsAsQualifiedNameSeq + ; + tableQualifiedName : tableIdent (DOT ident)* # identsAsTableQualifiedName ; @@ -1352,11 +1075,6 @@ wcQualifiedName : wildcard (DOT wildcard)* # identsAsWildcardQualifiedName ; -identifierSeq - : qualifiedName (COMMA qualifiedName)* # identsAsQualifiedNameSeq - | LT_PRTHS qualifiedName (COMMA qualifiedName)* RT_PRTHS # identsAsQualifiedNameSeq - ; - ident : (DOT)? ID | BACKTICK ident BACKTICK @@ -1376,49 +1094,40 @@ wildcard ; keywordsCanBeId - : searchableKeyWord - | IN - ; - -searchableKeyWord : D // OD SQL and ODBC special | timespanUnit | SPAN | evalFunctionName - | jsonFunctionName | relevanceArgName | intervalUnit - | trendlineType + | dateTimeFunctionName + | textFunctionName + | jsonFunctionName + | mathematicalFunctionName + | positionFunctionName + | cryptographicFunctionName | singleFieldRelevanceFunctionName | multiFieldRelevanceFunctionName | commandName - | collectionFunctionName - | REGEX + | comparisonOperator | explainMode - | REGEXP + | correlationType + | geoIpProperty // commands assist keywords - | CASE - | ELSE + | GEOIP + | OVERRIDE | ARROW - | BETWEEN - | EXISTS + | IN | SOURCE | INDEX - | A - | ASC | DESC | DATASOURCES | FROM | PATTERN | NEW_FIELD - | METHOD - | VARIABLE_COUNT_THRESHOLD - | FREQUENCY_THRESHOLD_PERCENTAGE - | MAX_SAMPLE_COUNT - | BUFFER_LIMIT + | SCOPE + | MAPPING | WITH - | REGEX - | PUNCT | USING | CAST | GET_FORMAT @@ -1426,12 +1135,8 @@ searchableKeyWord | INTERVAL | PLUS | MINUS - | OVERRIDE - // SORT FIELD KEYWORDS - | AUTO - | STR - | IP - | NUM + | INCLUDEFIELDS + | NULLS // ARGUMENT KEYWORDS | KEEPEMPTY | CONSECUTIVE @@ -1439,7 +1144,6 @@ searchableKeyWord | PARTITIONS | ALLNUM | DELIM - | BUCKET_NULLABLE | CENTROIDS | ITERATIONS | DISTANCE_TYPE @@ -1454,17 +1158,12 @@ searchableKeyWord | TIME_ZONE | TRAINING_DATA_SIZE | ANOMALY_SCORE_THRESHOLD - | COUNTFIELD - | SHOWCOUNT - | PATH - | INPUT - | OUTPUT - - // AGGREGATIONS AND WINDOW + // AGGREGATIONS | statsFunctionName - | windowFunctionName | DISTINCT_COUNT | DISTINCT_COUNT_APPROX + | PERCENTILE + | PERCENTILE_APPROX | ESTDC | ESTDC_ERROR | MEAN @@ -1477,6 +1176,8 @@ searchableKeyWord | VAR_SAMP | VAR_POP | TAKE + | FIRST + | LAST | LIST | VALUES | PER_DAY @@ -1496,7 +1197,12 @@ searchableKeyWord | FULL | SEMI | ANTI - | LEFT_HINT - | RIGHT_HINT - | PERCENTILE_SHORTCUT + | BETWEEN + | CIDRMATCH + | trendlineType + // SORT FIELD KEYWORDS + | AUTO + | STR + | IP + | NUM ; diff --git a/opensearch/build.gradle b/opensearch/build.gradle index 27aa81b0b67..2bf5e9cf2dd 100644 --- a/opensearch/build.gradle +++ b/opensearch/build.gradle @@ -41,6 +41,10 @@ dependencies { compileOnly group: 'org.opensearch.client', name: 'opensearch-rest-high-level-client', version: "${opensearch_version}" implementation group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}" implementation group: 'org.opensearch', name:'geospatial-client', version: "${opensearch_build}" + implementation 'io.substrait:core:0.67.0' + implementation('io.substrait:isthmus:0.67.0') { + exclude group: 'org.apache.calcite' + } annotationProcessor 'org.immutables:value:2.8.8' compileOnly 'org.immutables:value-annotations:2.8.8' diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java index 51a4db60ffd..2548c67cffe 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java @@ -42,7 +42,8 @@ public enum MappingType { HalfFloat("half_float", ExprCoreType.FLOAT), ScaledFloat("scaled_float", ExprCoreType.DOUBLE), Double("double", ExprCoreType.DOUBLE), - Boolean("boolean", ExprCoreType.BOOLEAN); + Boolean("boolean", ExprCoreType.BOOLEAN), + Alias("alias", ExprCoreType.UNKNOWN); // TODO: ranges, geo shape, point, shape private final String name; @@ -117,12 +118,7 @@ public static Map parseMapping(Map i // by default, the type is treated as an Object if "type" is not provided var type = ((String) innerMap.getOrDefault("type", "object")).replace("_", ""); if (!EnumUtils.isValidEnumIgnoreCase(OpenSearchDataType.MappingType.class, type)) { - // unknown type, e.g. `alias` - // Record fields of the alias type and resolve them later in case their references have - // not been resolved. - if (OpenSearchAliasType.typeName.equals(type)) { - aliasMapping.put(k, (String) innerMap.get(OpenSearchAliasType.pathPropertyName)); - } + // unknown type, skip it. return; } // create OpenSearchDataType @@ -133,21 +129,6 @@ public static Map parseMapping(Map i innerMap)); }); - // Begin to parse alias type fields - if (!aliasMapping.isEmpty()) { - // The path of alias type may point to a nested field, so we need to flatten the result. - Map flattenResult = traverseAndFlatten(result); - aliasMapping.forEach( - (k, v) -> { - if (flattenResult.containsKey(v)) { - result.put(k, new OpenSearchAliasType(v, flattenResult.get(v))); - } else { - throw new IllegalStateException( - String.format("Cannot find the path [%s] for alias type field [%s]", v, k)); - } - }); - } - return result; } @@ -188,6 +169,10 @@ public static OpenSearchDataType of(MappingType mappingType, Map // Default date formatter is used when "" is passed as the second parameter String format = (String) innerMap.getOrDefault("format", ""); return OpenSearchDateType.of(format); + case Alias: + return new OpenSearchAliasType( + (String) innerMap.get(OpenSearchAliasType.pathPropertyName), + OpenSearchDateType.of(MappingType.Invalid)); default: return res; } @@ -302,9 +287,20 @@ public void accept(Map subtree, String prefix) { } }; visitLevel.accept(tree, ""); + validateAliasType(result); return result; } + private static void validateAliasType(Map result) { + result.forEach( + (key, value) -> { + if (value instanceof OpenSearchAliasType && value.getOriginalPath().isPresent()) { + String originalPath = value.getOriginalPath().get(); + result.put(key, new OpenSearchAliasType(originalPath, result.get(originalPath))); + } + }); + } + /** * Resolve type of identified from parsed mapping tree. * diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 6fb08c6ab69..998c0f19ef5 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -85,11 +85,7 @@ public class OpenSearchExprValueFactory { * @param typeMapping A data type mapping produced by aggregation. */ public void extendTypeMapping(Map typeMapping) { - for (var field : typeMapping.keySet()) { - // Prevent overwriting, because aggregation engine may be not aware - // of all niceties of all types. - this.typeMapping.putIfAbsent(field, typeMapping.get(field)); - } + this.typeMapping.putAll(typeMapping); } @Getter @Setter private OpenSearchAggregationResponseParser parser; @@ -194,7 +190,12 @@ private ExprValue parse( // Field type may be not defined in mapping if users have disabled dynamic mapping. // Then try to parse content directly based on the value itself - if (fieldType.isEmpty()) { + // Besides, sub-fields of generated objects are also of type UNDEFINED. We parse the content + // directly on the value itself for this case as well. + // TODO: Remove the second condition once https://github.com/opensearch-project/sql/issues/3751 + // is resolved + if (fieldType.isEmpty() + || fieldType.get().equals(OpenSearchDataType.of(ExprCoreType.UNDEFINED))) { return parseContent(content); } @@ -250,7 +251,7 @@ private ExprValue parseContent(Content content) { * In OpenSearch, it is possible field doesn't have type definition in mapping. but has empty * value. For example, {"empty_field": []}. */ - private Optional type(String field) { + public Optional type(String field) { return Optional.ofNullable(typeMapping.get(field)); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java index b5c2d6edccc..6f0d4bf2f5a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java @@ -5,6 +5,7 @@ package org.opensearch.sql.opensearch.executor; +import com.google.common.base.Suppliers; import java.security.AccessController; import java.security.PrivilegedAction; import java.sql.PreparedStatement; @@ -15,7 +16,9 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelRoot; @@ -23,8 +26,11 @@ import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.runtime.Hook; import org.apache.calcite.sql.SqlExplainLevel; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlOperatorTable; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.util.ListSqlOperatorTable; import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction; import org.apache.calcite.sql.validate.SqlUserDefinedFunction; import org.apache.logging.log4j.LogManager; @@ -205,7 +211,8 @@ public void execute( () -> { try (PreparedStatement statement = OpenSearchRelRunners.run(context, rel)) { ResultSet result = statement.executeQuery(); - buildResultSet(result, rel.getRowType(), context.querySizeLimit, listener); + buildResultSet( + result, rel.getRowType(), context.sysLimit.querySizeLimit(), listener); } catch (SQLException e) { throw new RuntimeException(e); } @@ -270,8 +277,9 @@ private void buildResultSet( private void registerOpenSearchFunctions() { if (client instanceof OpenSearchNodeClient) { SqlUserDefinedFunction geoIpFunction = - new GeoIpFunction(client.getNodeClient()).toUDF("GEOIP"); + new GeoIpFunction(client.getNodeClient()).toUDF(BuiltinFunctionName.GEOIP.name()); PPLFuncImpTable.INSTANCE.registerExternalOperator(BuiltinFunctionName.GEOIP, geoIpFunction); + OperatorTable.addOperator(BuiltinFunctionName.GEOIP.name(), geoIpFunction); } else { logger.info( "Function [GEOIP] not registered: incompatible client type {}", @@ -281,10 +289,37 @@ private void registerOpenSearchFunctions() { SqlUserDefinedAggFunction approxDistinctCountFunction = UserDefinedFunctionUtils.createUserDefinedAggFunction( DistinctCountApproxAggFunction.class, - "APPROX_DISTINCT_COUNT", + BuiltinFunctionName.DISTINCT_COUNT_APPROX.name(), ReturnTypes.BIGINT_FORCE_NULLABLE, null); PPLFuncImpTable.INSTANCE.registerExternalAggOperator( BuiltinFunctionName.DISTINCT_COUNT_APPROX, approxDistinctCountFunction); + OperatorTable.addOperator( + BuiltinFunctionName.DISTINCT_COUNT_APPROX.name(), approxDistinctCountFunction); + } + + /** + * Dynamic SqlOperatorTable that allows adding operators after initialization. Similar to + * PPLBuiltinOperator.instance() or SqlStdOperatorTable.instance(). + */ + public static class OperatorTable extends ListSqlOperatorTable { + private static final Supplier INSTANCE = + Suppliers.memoize(() -> (OperatorTable) new OperatorTable().init()); + // Use map instead of list to avoid duplicated elements if the class is initialized multiple + // times + private static final Map operators = new ConcurrentHashMap<>(); + + public static SqlOperatorTable instance() { + return INSTANCE.get(); + } + + private ListSqlOperatorTable init() { + setOperators(buildIndex(operators.values())); + return this; + } + + public static synchronized void addOperator(String name, SqlOperator operator) { + operators.put(name, operator); + } } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java index 9b3c4b0a1f5..001ed064d1d 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java @@ -16,16 +16,15 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; -import org.apache.calcite.sql.type.CompositeOperandTypeChecker; -import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.geospatial.action.IpEnrichmentActionClient; import org.opensearch.sql.common.utils.StringUtils; +import org.opensearch.sql.data.model.ExprIpValue; import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; import org.opensearch.transport.client.node.NodeClient; @@ -38,8 +37,8 @@ *

Signatures: * *

    - *
  • (STRING, STRING) -> MAP - *
  • (STRING, STRING, STRING) -> MAP + *
  • (STRING, IP) -> MAP + *
  • (STRING, IP, STRING) -> MAP *
*/ public class GeoIpFunction extends ImplementorUDF { @@ -59,11 +58,10 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.CHARACTER_CHARACTER.or( - OperandTypes.family( - SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER))); + return UDFOperandMetadata.wrapUDT( + List.of( + List.of(ExprCoreType.STRING, ExprCoreType.IP), + List.of(ExprCoreType.STRING, ExprCoreType.IP, ExprCoreType.STRING))); } public static class GeoIPImplementor implements NotNullImplementor { @@ -87,16 +85,20 @@ public Expression implement( } public static Map fetchIpEnrichment( - String dataSource, String ipAddress, NodeClient nodeClient) { - return fetchIpEnrichment(dataSource, ipAddress, Collections.emptySet(), nodeClient); + String dataSource, ExprIpValue ipAddress, NodeClient nodeClient) { + return fetchIpEnrichment( + dataSource, ipAddress.toString(), Collections.emptySet(), nodeClient); } public static Map fetchIpEnrichment( - String dataSource, String ipAddress, String commaSeparatedOptions, NodeClient nodeClient) { + String dataSource, + ExprIpValue ipAddress, + String commaSeparatedOptions, + NodeClient nodeClient) { String unquotedOptions = StringUtils.unquoteText(commaSeparatedOptions); final Set options = Arrays.stream(unquotedOptions.split(",")).map(String::trim).collect(Collectors.toSet()); - return fetchIpEnrichment(dataSource, ipAddress, options, nodeClient); + return fetchIpEnrichment(dataSource, ipAddress.toString(), options, nodeClient); } private static Map fetchIpEnrichment( diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchAggregateIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchAggregateIndexScanRule.java deleted file mode 100644 index f5f9969b8fc..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchAggregateIndexScanRule.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.planner.physical; - -import static org.opensearch.sql.expression.function.PPLBuiltinOperators.WIDTH_BUCKET; - -import java.util.List; -import java.util.function.Predicate; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.plan.RelRule; -import org.apache.calcite.rel.AbstractRelNode; -import org.apache.calcite.rel.logical.LogicalAggregate; -import org.apache.calcite.rel.logical.LogicalProject; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.RexCall; -import org.apache.calcite.sql.SqlKind; -import org.immutables.value.Value; -import org.opensearch.sql.calcite.type.ExprSqlType; -import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT; -import org.opensearch.sql.opensearch.storage.scan.CalciteLogicalIndexScan; - -/** Planner rule that push a {@link LogicalAggregate} down to {@link CalciteLogicalIndexScan} */ -@Value.Enclosing -public class OpenSearchAggregateIndexScanRule - extends RelRule { - - /** Creates a OpenSearchAggregateIndexScanRule. */ - protected OpenSearchAggregateIndexScanRule(Config config) { - super(config); - } - - @Override - public void onMatch(RelOptRuleCall call) { - if (call.rels.length == 3) { - final LogicalAggregate aggregate = call.rel(0); - final LogicalProject project = call.rel(1); - final CalciteLogicalIndexScan scan = call.rel(2); - - // For multiple group-by, we currently have to use CompositeAggregationBuilder while it - // doesn't support auto_date_histogram referring to bin command with parameter bins - if (aggregate.getGroupSet().length() > 1 && Config.containsWidthBucketFuncOnDate(project)) { - return; - } - - apply(call, aggregate, project, scan); - } else if (call.rels.length == 2) { - // case of count() without group-by - final LogicalAggregate aggregate = call.rel(0); - final CalciteLogicalIndexScan scan = call.rel(1); - apply(call, aggregate, null, scan); - } else { - throw new AssertionError( - String.format( - "The length of rels should be %s but got %s", - this.operands.size(), call.rels.length)); - } - } - - protected void apply( - RelOptRuleCall call, - LogicalAggregate aggregate, - LogicalProject project, - CalciteLogicalIndexScan scan) { - AbstractRelNode newRelNode = scan.pushDownAggregate(aggregate, project); - if (newRelNode != null) { - call.transformTo(newRelNode); - } - } - - /** Rule configuration. */ - @Value.Immutable - public interface Config extends RelRule.Config { - Config DEFAULT = - ImmutableOpenSearchAggregateIndexScanRule.Config.builder() - .build() - .withDescription("Agg-Project-TableScan") - .withOperandSupplier( - b0 -> - b0.operand(LogicalAggregate.class) - .oneInput( - b1 -> - b1.operand(LogicalProject.class) - .predicate( - // Support push down aggregate with project that: - // 1. No RexOver and no duplicate projection - // 2. Contains width_bucket function on date field referring - // to bin command with parameter bins - Predicate.not(OpenSearchIndexScanRule::containsRexOver) - .and(OpenSearchIndexScanRule::distinctProjectList) - .or(Config::containsWidthBucketFuncOnDate)) - .oneInput( - b2 -> - b2.operand(CalciteLogicalIndexScan.class) - .predicate( - Predicate.not( - OpenSearchIndexScanRule::isLimitPushed) - .and( - OpenSearchIndexScanRule - ::noAggregatePushed)) - .noInputs()))); - Config COUNT_STAR = - ImmutableOpenSearchAggregateIndexScanRule.Config.builder() - .build() - .withDescription("Agg[count()]-TableScan") - .withOperandSupplier( - b0 -> - b0.operand(LogicalAggregate.class) - .predicate( - agg -> - agg.getGroupSet().isEmpty() - && agg.getAggCallList().stream() - .allMatch( - call -> - call.getAggregation().kind == SqlKind.COUNT - && call.getArgList().isEmpty())) - .oneInput( - b1 -> - b1.operand(CalciteLogicalIndexScan.class) - .predicate( - Predicate.not(OpenSearchIndexScanRule::isLimitPushed) - .and(OpenSearchIndexScanRule::noAggregatePushed)) - .noInputs())); - - @Override - default OpenSearchAggregateIndexScanRule toRule() { - return new OpenSearchAggregateIndexScanRule(this); - } - - static boolean containsWidthBucketFuncOnDate(LogicalProject project) { - return project.getProjects().stream() - .anyMatch( - expr -> - expr instanceof RexCall rexCall - && rexCall.getOperator().equals(WIDTH_BUCKET) - && dateRelatedType(rexCall.getOperands().getFirst().getType())); - } - - static boolean dateRelatedType(RelDataType type) { - return type instanceof ExprSqlType exprSqlType - && List.of(ExprUDT.EXPR_DATE, ExprUDT.EXPR_TIME, ExprUDT.EXPR_TIMESTAMP) - .contains(exprSqlType.getUdt()); - } - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchIndexRules.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchIndexRules.java deleted file mode 100644 index 6d349aa452d..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchIndexRules.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.planner.physical; - -import com.google.common.collect.ImmutableList; -import java.util.List; -import org.apache.calcite.plan.RelOptRule; - -public class OpenSearchIndexRules { - private static final OpenSearchProjectIndexScanRule PROJECT_INDEX_SCAN = - OpenSearchProjectIndexScanRule.Config.DEFAULT.toRule(); - private static final OpenSearchFilterIndexScanRule FILTER_INDEX_SCAN = - OpenSearchFilterIndexScanRule.Config.DEFAULT.toRule(); - private static final OpenSearchAggregateIndexScanRule AGGREGATE_INDEX_SCAN = - OpenSearchAggregateIndexScanRule.Config.DEFAULT.toRule(); - private static final OpenSearchAggregateIndexScanRule COUNT_STAR_INDEX_SCAN = - OpenSearchAggregateIndexScanRule.Config.COUNT_STAR.toRule(); - private static final OpenSearchLimitIndexScanRule LIMIT_INDEX_SCAN = - OpenSearchLimitIndexScanRule.Config.DEFAULT.toRule(); - private static final OpenSearchSortIndexScanRule SORT_INDEX_SCAN = - OpenSearchSortIndexScanRule.Config.DEFAULT.toRule(); - private static final OpenSearchDedupPushdownRule DEDUP_PUSH_DOWN = - OpenSearchDedupPushdownRule.Config.DEFAULT.toRule(); - private static final SortProjectExprTransposeRule SORT_PROJECT_EXPR_TRANSPOSE = - SortProjectExprTransposeRule.Config.DEFAULT.toRule(); - private static final ExpandCollationOnProjectExprRule EXPAND_COLLATION_ON_PROJECT_EXPR = - ExpandCollationOnProjectExprRule.Config.DEFAULT.toRule(); - - // Rule that always pushes down relevance functions regardless of pushdown settings - public static final OpenSearchRelevanceFunctionPushdownRule RELEVANCE_FUNCTION_PUSHDOWN = - OpenSearchRelevanceFunctionPushdownRule.Config.DEFAULT.toRule(); - - public static final List OPEN_SEARCH_INDEX_SCAN_RULES = - ImmutableList.of( - PROJECT_INDEX_SCAN, - FILTER_INDEX_SCAN, - AGGREGATE_INDEX_SCAN, - COUNT_STAR_INDEX_SCAN, - LIMIT_INDEX_SCAN, - SORT_INDEX_SCAN, - // TODO enable if https://github.com/opensearch-project/OpenSearch/issues/3725 resolved - // DEDUP_PUSH_DOWN, - SORT_PROJECT_EXPR_TRANSPOSE, - EXPAND_COLLATION_ON_PROJECT_EXPR); - - // prevent instantiation - private OpenSearchIndexRules() {} -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchIndexScanRule.java deleted file mode 100644 index 24abb3c3bc9..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchIndexScanRule.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.planner.physical; - -import java.util.HashSet; -import java.util.Set; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.Sort; -import org.apache.calcite.rel.logical.LogicalProject; -import org.apache.calcite.rel.logical.LogicalSort; -import org.apache.calcite.rex.RexCall; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexOver; -import org.apache.calcite.util.Pair; -import org.opensearch.sql.opensearch.storage.OpenSearchIndex; -import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan; - -public interface OpenSearchIndexScanRule { - /** - * CalciteOpenSearchIndexScan doesn't allow push-down anymore (except Sort under some strict - * condition) after Aggregate push-down. - */ - static boolean noAggregatePushed(AbstractCalciteIndexScan scan) { - if (scan.getPushDownContext().isAggregatePushed()) return false; - final RelOptTable table = scan.getTable(); - return table.unwrap(OpenSearchIndex.class) != null; - } - - static boolean isLimitPushed(AbstractCalciteIndexScan scan) { - return scan.getPushDownContext().isLimitPushed(); - } - - // `RelDecorrelator` may generate a Project with duplicated fields, e.g. Project($0,$0). - // There will be problem if pushing down the pattern like `Aggregate(AGG($0),{1})-Project($0,$0)`, - // as it will lead to field-name conflict. - // We should wait and rely on `AggregateProjectMergeRule` to mitigate it by having this constraint - // Nevertheless, that rule cannot handle all cases if there is RexCall in the Project, - // e.g. Project($0, $0, +($0,1)). We cannot push down the Aggregate for this corner case. - // TODO: Simplify the Project where there is RexCall by adding a new rule. - static boolean distinctProjectList(LogicalProject project) { - // Change to Set> to resolve - // https://github.com/opensearch-project/sql/issues/4347 - Set> rexSet = new HashSet<>(); - return project.getNamedProjects().stream().allMatch(rexSet::add); - } - - static boolean containsRexOver(LogicalProject project) { - return project.getProjects().stream().anyMatch(RexOver::containsOver); - } - - /** - * The LogicalSort is a LIMIT that should be pushed down when its fetch field is not null and its - * collation is empty. For example: sort name | head 5 should not be pushed down - * because it has a field collation. - * - * @param sort The LogicalSort to check. - * @return True if the LogicalSort is a LIMIT, false otherwise. - */ - static boolean isLogicalSortLimit(LogicalSort sort) { - return sort.fetch != null; - } - - static boolean projectContainsExpr(Project project) { - return project.getProjects().stream().anyMatch(p -> p instanceof RexCall); - } - - static boolean sortByFieldsOnly(Sort sort) { - return !sort.getCollation().getFieldCollations().isEmpty() && sort.fetch == null; - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchSortIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchSortIndexScanRule.java deleted file mode 100644 index b95725c3e16..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchSortIndexScanRule.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.planner.physical; - -import java.util.function.Predicate; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.plan.RelRule; -import org.apache.calcite.rel.core.Sort; -import org.immutables.value.Value; -import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan; - -@Value.Enclosing -public class OpenSearchSortIndexScanRule extends RelRule { - - protected OpenSearchSortIndexScanRule(Config config) { - super(config); - } - - @Override - public void onMatch(RelOptRuleCall call) { - final Sort sort = call.rel(0); - final AbstractCalciteIndexScan scan = call.rel(1); - - var collations = sort.collation.getFieldCollations(); - AbstractCalciteIndexScan newScan = scan.pushDownSort(collations); - if (newScan != null) { - call.transformTo(newScan); - } - } - - /** Rule configuration. */ - @Value.Immutable - public interface Config extends RelRule.Config { - OpenSearchSortIndexScanRule.Config DEFAULT = - ImmutableOpenSearchSortIndexScanRule.Config.builder() - .build() - .withOperandSupplier( - b0 -> - b0.operand(Sort.class) - .predicate(OpenSearchIndexScanRule::sortByFieldsOnly) - .oneInput( - b1 -> - b1.operand(AbstractCalciteIndexScan.class) - // Skip the rule if a limit has already been pushed down - // because pushing down a sort after a limit will be treated - // as sort-then-limit by OpenSearch DSL. - .predicate( - Predicate.not(OpenSearchIndexScanRule::isLimitPushed)) - .noInputs())); - - @Override - default OpenSearchSortIndexScanRule toRule() { - return new OpenSearchSortIndexScanRule(this); - } - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/AggregateIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/AggregateIndexScanRule.java new file mode 100644 index 00000000000..462c4be7243 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/AggregateIndexScanRule.java @@ -0,0 +1,222 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.planner.rules; + +import static org.opensearch.sql.expression.function.PPLBuiltinOperators.WIDTH_BUCKET; + +import java.util.List; +import java.util.function.Function; +import java.util.function.Predicate; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.AbstractRelNode; +import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.logical.LogicalFilter; +import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.immutables.value.Value; +import org.opensearch.sql.ast.expression.Argument; +import org.opensearch.sql.calcite.utils.PlanUtils; +import org.opensearch.sql.expression.function.udf.binning.WidthBucketFunction; +import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan; +import org.opensearch.sql.opensearch.storage.scan.CalciteLogicalIndexScan; + +/** Planner rule that push a {@link LogicalAggregate} down to {@link CalciteLogicalIndexScan} */ +@Value.Enclosing +public class AggregateIndexScanRule extends RelRule { + + /** Creates a AggregateIndexScanRule. */ + protected AggregateIndexScanRule(Config config) { + super(config); + } + + @Override + public void onMatch(RelOptRuleCall call) { + if (call.rels.length == 4) { + final LogicalAggregate aggregate = call.rel(0); + final LogicalFilter filter = call.rel(1); + final LogicalProject project = call.rel(2); + final CalciteLogicalIndexScan scan = call.rel(3); + List groupSet = aggregate.getGroupSet().asList(); + RexNode condition = filter.getCondition(); + Function isNotNullFromAgg = + rex -> + rex instanceof RexCall rexCall + && rexCall.getOperator() == SqlStdOperatorTable.IS_NOT_NULL + && rexCall.getOperands().get(0) instanceof RexInputRef ref + && groupSet.contains(ref.getIndex()); + if (isNotNullFromAgg.apply(condition) + || (condition instanceof RexCall rexCall + && rexCall.getOperator() == SqlStdOperatorTable.AND + && rexCall.getOperands().stream().allMatch(isNotNullFromAgg::apply))) { + // Try to do the aggregate push down and ignore the filter if the filter sources from the + // aggregate's hint. See{@link CalciteRelNodeVisitor::visitAggregation} + apply(call, aggregate, project, scan); + } + } else if (call.rels.length == 3) { + final LogicalAggregate aggregate = call.rel(0); + final LogicalProject project = call.rel(1); + final CalciteLogicalIndexScan scan = call.rel(2); + apply(call, aggregate, project, scan); + } else if (call.rels.length == 2) { + // case of count() without group-by + final LogicalAggregate aggregate = call.rel(0); + final CalciteLogicalIndexScan scan = call.rel(1); + apply(call, aggregate, null, scan); + } else { + throw new AssertionError( + String.format( + "The length of rels should be %s but got %s", + this.operands.size(), call.rels.length)); + } + } + + protected void apply( + RelOptRuleCall call, + LogicalAggregate aggregate, + LogicalProject project, + CalciteLogicalIndexScan scan) { + AbstractRelNode newRelNode = scan.pushDownAggregate(aggregate, project); + if (newRelNode != null) { + call.transformTo(newRelNode); + } + } + + /** Rule configuration. */ + @Value.Immutable + public interface Config extends RelRule.Config { + Config DEFAULT = + ImmutableAggregateIndexScanRule.Config.builder() + .build() + .withDescription("Agg-Project-TableScan") + .withOperandSupplier( + b0 -> + b0.operand(LogicalAggregate.class) + .oneInput( + b1 -> + b1.operand(LogicalProject.class) + .predicate( + // Support push down aggregate with project that: + // 1. No RexOver and no duplicate projection + // 2. Contains width_bucket function on date field referring + // to bin command with parameter bins + Predicate.not(PlanUtils::containsRexOver) + .and(PlanUtils::distinctProjectList) + .or(Config::containsWidthBucketFuncOnDate)) + .oneInput( + b2 -> + b2.operand(CalciteLogicalIndexScan.class) + .predicate( + Predicate.not( + AbstractCalciteIndexScan::isLimitPushed) + .and( + AbstractCalciteIndexScan + ::noAggregatePushed)) + .noInputs()))); + Config COUNT_STAR = + ImmutableAggregateIndexScanRule.Config.builder() + .build() + .withDescription("Agg[count()]-TableScan") + .withOperandSupplier( + b0 -> + b0.operand(LogicalAggregate.class) + .predicate( + agg -> + agg.getGroupSet().isEmpty() + && agg.getAggCallList().stream() + .allMatch( + call -> + call.getAggregation().kind == SqlKind.COUNT + && call.getArgList().isEmpty())) + .oneInput( + b1 -> + b1.operand(CalciteLogicalIndexScan.class) + .predicate( + Predicate.not(AbstractCalciteIndexScan::isLimitPushed) + .and(AbstractCalciteIndexScan::noAggregatePushed)) + .noInputs())); + // TODO: No need this rule once https://github.com/opensearch-project/sql/issues/4403 is + // addressed + Config BUCKET_NON_NULL_AGG = + ImmutableAggregateIndexScanRule.Config.builder() + .build() + .withDescription("Agg-Filter-Project-TableScan") + .withOperandSupplier( + b0 -> + b0.operand(LogicalAggregate.class) + .predicate( + agg -> + agg.getHints().stream() + .anyMatch( + hint -> + hint.hintName.equals("stats_args") + && hint.kvOptions + .get(Argument.BUCKET_NULLABLE) + .equals("false"))) + .oneInput( + b1 -> + b1.operand(LogicalFilter.class) + .predicate(Config::mayBeFilterFromBucketNonNull) + .oneInput( + b2 -> + b2.operand(LogicalProject.class) + .predicate( + // Support push down aggregate with project + // that: + // 1. No RexOver and no duplicate projection + // 2. Contains width_bucket function on date + // field referring + // to bin command with parameter bins + Predicate.not(PlanUtils::containsRexOver) + .and(PlanUtils::distinctProjectList) + .or(Config::containsWidthBucketFuncOnDate)) + .oneInput( + b3 -> + b3.operand(CalciteLogicalIndexScan.class) + .predicate( + Predicate.not( + AbstractCalciteIndexScan + ::isLimitPushed) + .and( + AbstractCalciteIndexScan + ::noAggregatePushed)) + .noInputs())))); + + @Override + default AggregateIndexScanRule toRule() { + return new AggregateIndexScanRule(this); + } + + static boolean mayBeFilterFromBucketNonNull(LogicalFilter filter) { + RexNode condition = filter.getCondition(); + return isNotNullOnRef(condition) + || (condition instanceof RexCall rexCall + && rexCall.getOperator().equals(SqlStdOperatorTable.AND) + && rexCall.getOperands().stream() + .allMatch(AggregateIndexScanRule.Config::isNotNullOnRef)); + } + + private static boolean isNotNullOnRef(RexNode rex) { + return rex instanceof RexCall rexCall + && rexCall.getOperator().equals(SqlStdOperatorTable.IS_NOT_NULL) + && rexCall.getOperands().get(0) instanceof RexInputRef; + } + + static boolean containsWidthBucketFuncOnDate(LogicalProject project) { + return project.getProjects().stream() + .anyMatch( + expr -> + expr instanceof RexCall rexCall + && rexCall.getOperator().equals(WIDTH_BUCKET) + && WidthBucketFunction.dateRelatedType( + rexCall.getOperands().getFirst().getType())); + } + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchDedupPushdownRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/DedupPushdownRule.java similarity index 91% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchDedupPushdownRule.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/DedupPushdownRule.java index a51fa365905..9adc14ee70f 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchDedupPushdownRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/DedupPushdownRule.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.opensearch.planner.physical; +package org.opensearch.sql.opensearch.planner.rules; import static org.opensearch.sql.calcite.utils.PlanUtils.ROW_NUMBER_COLUMN_FOR_DEDUP; @@ -23,13 +23,14 @@ import org.apache.logging.log4j.Logger; import org.immutables.value.Value; import org.opensearch.sql.calcite.utils.PlanUtils; +import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan; import org.opensearch.sql.opensearch.storage.scan.CalciteLogicalIndexScan; @Value.Enclosing -public class OpenSearchDedupPushdownRule extends RelRule { +public class DedupPushdownRule extends RelRule { private static final Logger LOG = LogManager.getLogger(); - protected OpenSearchDedupPushdownRule(Config config) { + protected DedupPushdownRule(Config config) { super(config); } @@ -107,7 +108,7 @@ private static boolean validFilter(LogicalFilter filter) { @Value.Immutable public interface Config extends RelRule.Config { Config DEFAULT = - ImmutableOpenSearchDedupPushdownRule.Config.builder() + ImmutableDedupPushdownRule.Config.builder() .build() .withOperandSupplier( b0 -> @@ -115,7 +116,7 @@ public interface Config extends RelRule.Config { .oneInput( b1 -> b1.operand(LogicalFilter.class) - .predicate(OpenSearchDedupPushdownRule::validFilter) + .predicate(DedupPushdownRule::validFilter) .oneInput( b2 -> b2.operand(LogicalProject.class) @@ -125,16 +126,16 @@ public interface Config extends RelRule.Config { b3.operand(CalciteLogicalIndexScan.class) .predicate( Predicate.not( - OpenSearchIndexScanRule + AbstractCalciteIndexScan ::isLimitPushed) .and( - OpenSearchIndexScanRule + AbstractCalciteIndexScan ::noAggregatePushed)) .noInputs())))); @Override - default OpenSearchDedupPushdownRule toRule() { - return new OpenSearchDedupPushdownRule(this); + default DedupPushdownRule toRule() { + return new DedupPushdownRule(this); } } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/EnumerableIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableIndexScanRule.java similarity index 97% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/EnumerableIndexScanRule.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableIndexScanRule.java index 8dc7f3e187b..4a6ac7fff7b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/EnumerableIndexScanRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableIndexScanRule.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.opensearch.planner.physical; +package org.opensearch.sql.opensearch.planner.rules; import org.apache.calcite.adapter.enumerable.EnumerableConvention; import org.apache.calcite.plan.Convention; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/EnumerableSystemIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java similarity index 96% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/EnumerableSystemIndexScanRule.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java index da277c2af4c..616d1178873 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/EnumerableSystemIndexScanRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.opensearch.planner.physical; +package org.opensearch.sql.opensearch.planner.rules; import org.apache.calcite.adapter.enumerable.EnumerableConvention; import org.apache.calcite.plan.Convention; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/ExpandCollationOnProjectExprRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/ExpandCollationOnProjectExprRule.java similarity index 96% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/ExpandCollationOnProjectExprRule.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/ExpandCollationOnProjectExprRule.java index 36acc3b0dab..b1bd711601d 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/ExpandCollationOnProjectExprRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/ExpandCollationOnProjectExprRule.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.opensearch.planner.physical; +package org.opensearch.sql.opensearch.planner.rules; import java.util.Optional; import org.apache.calcite.adapter.enumerable.EnumerableProject; @@ -19,6 +19,7 @@ import org.apache.calcite.rel.core.Project; import org.apache.commons.lang3.tuple.Pair; import org.immutables.value.Value; +import org.opensearch.sql.calcite.utils.PlanUtils; import org.opensearch.sql.opensearch.util.OpenSearchRelOptUtil; /** @@ -108,7 +109,7 @@ public interface Config extends RelRule.Config { .oneInput( b1 -> b1.operand(EnumerableProject.class) - .predicate(OpenSearchIndexScanRule::projectContainsExpr) + .predicate(PlanUtils::projectContainsExpr) .predicate(p -> !p.containsOver()) .anyInputs())); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchFilterIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/FilterIndexScanRule.java similarity index 78% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchFilterIndexScanRule.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/FilterIndexScanRule.java index 306617ae2cf..8aa2f77b6ac 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchFilterIndexScanRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/FilterIndexScanRule.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.opensearch.planner.physical; +package org.opensearch.sql.opensearch.planner.rules; import java.util.function.Predicate; import org.apache.calcite.plan.RelOptRuleCall; @@ -12,14 +12,15 @@ import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.logical.LogicalFilter; import org.immutables.value.Value; +import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan; import org.opensearch.sql.opensearch.storage.scan.CalciteLogicalIndexScan; /** Planner rule that push a {@link LogicalFilter} down to {@link CalciteLogicalIndexScan} */ @Value.Enclosing -public class OpenSearchFilterIndexScanRule extends RelRule { +public class FilterIndexScanRule extends RelRule { - /** Creates a OpenSearchFilterIndexScanRule. */ - protected OpenSearchFilterIndexScanRule(Config config) { + /** Creates a FilterIndexScanRule. */ + protected FilterIndexScanRule(Config config) { super(config); } @@ -50,7 +51,7 @@ protected void apply(RelOptRuleCall call, Filter filter, CalciteLogicalIndexScan public interface Config extends RelRule.Config { /** Config that matches Filter on CalciteLogicalIndexScan. */ Config DEFAULT = - ImmutableOpenSearchFilterIndexScanRule.Config.builder() + ImmutableFilterIndexScanRule.Config.builder() .build() .withOperandSupplier( b0 -> @@ -64,13 +65,13 @@ public interface Config extends RelRule.Config { // handle filter pushdown after limit. Both "limit after // filter" and "filter after limit" result in the same // limit-after-filter DSL. - Predicate.not(OpenSearchIndexScanRule::isLimitPushed) - .and(OpenSearchIndexScanRule::noAggregatePushed)) + Predicate.not(AbstractCalciteIndexScan::isLimitPushed) + .and(AbstractCalciteIndexScan::noAggregatePushed)) .noInputs())); @Override - default OpenSearchFilterIndexScanRule toRule() { - return new OpenSearchFilterIndexScanRule(this); + default FilterIndexScanRule toRule() { + return new FilterIndexScanRule(this); } } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchLimitIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/LimitIndexScanRule.java similarity index 85% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchLimitIndexScanRule.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/LimitIndexScanRule.java index a6832b06a24..1c24c7664fc 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchLimitIndexScanRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/LimitIndexScanRule.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.opensearch.planner.physical; +package org.opensearch.sql.opensearch.planner.rules; import java.util.Objects; import org.apache.calcite.plan.RelOptRuleCall; @@ -13,6 +13,7 @@ import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.immutables.value.Value; +import org.opensearch.sql.calcite.utils.PlanUtils; import org.opensearch.sql.opensearch.storage.scan.CalciteLogicalIndexScan; /** @@ -20,9 +21,9 @@ * down to {@link CalciteLogicalIndexScan} */ @Value.Enclosing -public class OpenSearchLimitIndexScanRule extends RelRule { +public class LimitIndexScanRule extends RelRule { - protected OpenSearchLimitIndexScanRule(Config config) { + protected LimitIndexScanRule(Config config) { super(config); } @@ -82,18 +83,18 @@ private static Integer extractOffsetValue(RexNode offset) { /** Rule configuration. */ @Value.Immutable public interface Config extends RelRule.Config { - OpenSearchLimitIndexScanRule.Config DEFAULT = - ImmutableOpenSearchLimitIndexScanRule.Config.builder() + LimitIndexScanRule.Config DEFAULT = + ImmutableLimitIndexScanRule.Config.builder() .build() .withOperandSupplier( b0 -> b0.operand(LogicalSort.class) - .predicate(OpenSearchIndexScanRule::isLogicalSortLimit) + .predicate(PlanUtils::isLogicalSortLimit) .oneInput(b1 -> b1.operand(CalciteLogicalIndexScan.class).noInputs())); @Override - default OpenSearchLimitIndexScanRule toRule() { - return new OpenSearchLimitIndexScanRule(this); + default LimitIndexScanRule toRule() { + return new LimitIndexScanRule(this); } } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java new file mode 100644 index 00000000000..c7f007bbf49 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.planner.rules; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import org.apache.calcite.plan.RelOptRule; + +public class OpenSearchIndexRules { + private static final ProjectIndexScanRule PROJECT_INDEX_SCAN = + ProjectIndexScanRule.Config.DEFAULT.toRule(); + private static final FilterIndexScanRule FILTER_INDEX_SCAN = + FilterIndexScanRule.Config.DEFAULT.toRule(); + private static final AggregateIndexScanRule AGGREGATE_INDEX_SCAN = + AggregateIndexScanRule.Config.DEFAULT.toRule(); + private static final AggregateIndexScanRule COUNT_STAR_INDEX_SCAN = + AggregateIndexScanRule.Config.COUNT_STAR.toRule(); + // TODO: No need this rule once https://github.com/opensearch-project/sql/issues/4403 is addressed + private static final AggregateIndexScanRule BUCKET_NON_NULL_AGG_INDEX_SCAN = + AggregateIndexScanRule.Config.BUCKET_NON_NULL_AGG.toRule(); + private static final LimitIndexScanRule LIMIT_INDEX_SCAN = + LimitIndexScanRule.Config.DEFAULT.toRule(); + private static final SortIndexScanRule SORT_INDEX_SCAN = + SortIndexScanRule.Config.DEFAULT.toRule(); + private static final DedupPushdownRule DEDUP_PUSH_DOWN = + DedupPushdownRule.Config.DEFAULT.toRule(); + private static final SortProjectExprTransposeRule SORT_PROJECT_EXPR_TRANSPOSE = + SortProjectExprTransposeRule.Config.DEFAULT.toRule(); + private static final ExpandCollationOnProjectExprRule EXPAND_COLLATION_ON_PROJECT_EXPR = + ExpandCollationOnProjectExprRule.Config.DEFAULT.toRule(); + private static final SortAggregateMeasureRule SORT_AGGREGATION_METRICS_RULE = + SortAggregateMeasureRule.Config.DEFAULT.toRule(); + private static final RareTopPushdownRule RARE_TOP_PUSH_DOWN = + RareTopPushdownRule.Config.DEFAULT.toRule(); + + // Rule that always pushes down relevance functions regardless of pushdown settings + public static final RelevanceFunctionPushdownRule RELEVANCE_FUNCTION_PUSHDOWN = + RelevanceFunctionPushdownRule.Config.DEFAULT.toRule(); + + public static final List OPEN_SEARCH_INDEX_SCAN_RULES = + ImmutableList.of( + PROJECT_INDEX_SCAN, + FILTER_INDEX_SCAN, + AGGREGATE_INDEX_SCAN, + COUNT_STAR_INDEX_SCAN, + BUCKET_NON_NULL_AGG_INDEX_SCAN, + LIMIT_INDEX_SCAN, + SORT_INDEX_SCAN, + // TODO enable if https://github.com/opensearch-project/OpenSearch/issues/3725 resolved + // DEDUP_PUSH_DOWN, + SORT_PROJECT_EXPR_TRANSPOSE, + SORT_AGGREGATION_METRICS_RULE, + RARE_TOP_PUSH_DOWN, + EXPAND_COLLATION_ON_PROJECT_EXPR); + + // prevent instantiation + private OpenSearchIndexRules() {} +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchProjectIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/ProjectIndexScanRule.java similarity index 87% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchProjectIndexScanRule.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/ProjectIndexScanRule.java index b468d482570..d4c7986145e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchProjectIndexScanRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/ProjectIndexScanRule.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.opensearch.planner.physical; +package org.opensearch.sql.opensearch.planner.rules; import static java.util.Objects.requireNonNull; @@ -26,10 +26,10 @@ /** Planner rule that push a {@link LogicalProject} down to {@link CalciteLogicalIndexScan} */ @Value.Enclosing -public class OpenSearchProjectIndexScanRule extends RelRule { +public class ProjectIndexScanRule extends RelRule { - /** Creates a OpenSearchProjectIndexScanRule. */ - protected OpenSearchProjectIndexScanRule(Config config) { + /** Creates a ProjectIndexScanRule. */ + protected ProjectIndexScanRule(Config config) { super(config); } @@ -103,9 +103,9 @@ public boolean isIdentity(Integer size) { /** Rule configuration. */ @Value.Immutable public interface Config extends RelRule.Config { - /** Config that matches Project on OpenSearchProjectIndexScanRule. */ + /** Config that matches Project on ProjectIndexScanRule. */ Config DEFAULT = - ImmutableOpenSearchProjectIndexScanRule.Config.builder() + ImmutableProjectIndexScanRule.Config.builder() .build() .withOperandSupplier( b0 -> @@ -113,8 +113,8 @@ public interface Config extends RelRule.Config { .oneInput(b1 -> b1.operand(CalciteLogicalIndexScan.class).noInputs())); @Override - default OpenSearchProjectIndexScanRule toRule() { - return new OpenSearchProjectIndexScanRule(this); + default ProjectIndexScanRule toRule() { + return new ProjectIndexScanRule(this); } } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/RareTopPushdownRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/RareTopPushdownRule.java new file mode 100644 index 00000000000..a04d8cea0b2 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/RareTopPushdownRule.java @@ -0,0 +1,104 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.planner.rules; + +import java.util.List; +import java.util.function.Predicate; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.logical.LogicalFilter; +import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexFieldCollation; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexWindow; +import org.apache.calcite.sql.SqlKind; +import org.immutables.value.Value; +import org.opensearch.sql.calcite.utils.PlanUtils; +import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan; +import org.opensearch.sql.opensearch.storage.scan.CalciteLogicalIndexScan; +import org.opensearch.sql.opensearch.storage.scan.context.RareTopDigest; + +@Value.Enclosing +public class RareTopPushdownRule extends RelRule { + + protected RareTopPushdownRule(Config config) { + super(config); + } + + @Override + public void onMatch(RelOptRuleCall call) { + final LogicalFilter filter = call.rel(0); + final LogicalProject project = call.rel(1); + final CalciteLogicalIndexScan scan = call.rel(2); + RareTopDigest digest; + try { + RexLiteral numberLiteral = + (RexLiteral) ((RexCall) filter.getCondition()).getOperands().get(1); + Integer number = numberLiteral.getValueAs(Integer.class); + List windows = PlanUtils.getRexWindowFromProject(project); + if (windows.size() != 1) { + return; + } + final List fieldNameList = project.getInput().getRowType().getFieldNames(); + List groupIndices = PlanUtils.getSelectColumns(windows.getFirst().partitionKeys); + List byList = groupIndices.stream().map(fieldNameList::get).toList(); + + if (windows.getFirst().orderKeys.size() != 1) { + return; + } + RexFieldCollation orderKey = windows.getFirst().orderKeys.getFirst(); + List orderIndices = PlanUtils.getSelectColumns(List.of(orderKey.getKey())); + List orderList = orderIndices.stream().map(fieldNameList::get).toList(); + List targetList = + fieldNameList.stream() + .filter(Predicate.not(byList::contains)) + .filter(Predicate.not(orderList::contains)) + .toList(); + if (targetList.size() != 1) { + return; + } + String targetName = targetList.getFirst(); + digest = new RareTopDigest(targetName, byList, number, orderKey.getDirection()); + } catch (Exception e) { + return; + } + CalciteLogicalIndexScan newScan = scan.pushDownRareTop(project, digest); + if (newScan != null) { + call.transformTo(newScan); + } + } + + @Value.Immutable + public interface Config extends RelRule.Config { + RareTopPushdownRule.Config DEFAULT = + ImmutableRareTopPushdownRule.Config.builder() + .build() + .withDescription("Filter-Project(window)-TableScan(agg-pushed)") + .withOperandSupplier( + b0 -> + b0.operand(LogicalFilter.class) + .predicate( + filter -> filter.getCondition().getKind() == SqlKind.LESS_THAN_OR_EQUAL) + .oneInput( + b1 -> + b1.operand(LogicalProject.class) + .predicate(PlanUtils::containsRowNumberRareTop) + .oneInput( + b2 -> + b2.operand(CalciteLogicalIndexScan.class) + .predicate( + Predicate.not( + AbstractCalciteIndexScan + ::noAggregatePushed)) + .noInputs()))); + + @Override + default RareTopPushdownRule toRule() { + return new RareTopPushdownRule(this); + } + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchRelevanceFunctionPushdownRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/RelevanceFunctionPushdownRule.java similarity index 87% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchRelevanceFunctionPushdownRule.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/RelevanceFunctionPushdownRule.java index c36a410dccf..31a67f49757 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchRelevanceFunctionPushdownRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/RelevanceFunctionPushdownRule.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.opensearch.planner.physical; +package org.opensearch.sql.opensearch.planner.rules; import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.MULTI_FIELDS_RELEVANCE_FUNCTION_SET; import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.SINGLE_FIELD_RELEVANCE_FUNCTION_SET; @@ -26,11 +26,10 @@ * relevance functions are always executed by OpenSearch for optimal performance and functionality. */ @Value.Enclosing -public class OpenSearchRelevanceFunctionPushdownRule - extends RelRule { +public class RelevanceFunctionPushdownRule extends RelRule { - /** Creates an OpenSearchRelevanceFunctionPushdownRule. */ - protected OpenSearchRelevanceFunctionPushdownRule(Config config) { + /** Creates an RelevanceFunctionPushdownRule. */ + protected RelevanceFunctionPushdownRule(Config config) { super(config); } @@ -104,7 +103,7 @@ boolean hasRelevanceFunction() { public interface Config extends RelRule.Config { /** Config that matches Filter on CalciteLogicalIndexScan. */ Config DEFAULT = - ImmutableOpenSearchRelevanceFunctionPushdownRule.Config.builder() + ImmutableRelevanceFunctionPushdownRule.Config.builder() .build() .withOperandSupplier( b0 -> @@ -112,8 +111,8 @@ public interface Config extends RelRule.Config { .oneInput(b1 -> b1.operand(CalciteLogicalIndexScan.class).noInputs())); @Override - default OpenSearchRelevanceFunctionPushdownRule toRule() { - return new OpenSearchRelevanceFunctionPushdownRule(this); + default RelevanceFunctionPushdownRule toRule() { + return new RelevanceFunctionPushdownRule(this); } } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/SortAggregateMeasureRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/SortAggregateMeasureRule.java new file mode 100644 index 00000000000..1b40063e6b1 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/SortAggregateMeasureRule.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.planner.rules; + +import java.util.function.Predicate; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.core.Sort; +import org.apache.calcite.rel.logical.LogicalSort; +import org.immutables.value.Value; +import org.opensearch.sql.calcite.utils.PlanUtils; +import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan; +import org.opensearch.sql.opensearch.storage.scan.CalciteLogicalIndexScan; + +@Value.Enclosing +public class SortAggregateMeasureRule extends RelRule { + + protected SortAggregateMeasureRule(Config config) { + super(config); + } + + @Override + public void onMatch(RelOptRuleCall call) { + final LogicalSort sort = call.rel(0); + final CalciteLogicalIndexScan scan = call.rel(1); + CalciteLogicalIndexScan newScan = scan.pushDownSortAggregateMeasure(sort); + if (newScan != null) { + call.transformTo(newScan); + } + } + + /** Rule configuration. */ + @Value.Immutable + public interface Config extends RelRule.Config { + // TODO support multiple measures, only support single measure sort + Predicate hasOneFieldCollation = + sort -> sort.getCollation().getFieldCollations().size() == 1; + SortAggregateMeasureRule.Config DEFAULT = + ImmutableSortAggregateMeasureRule.Config.builder() + .build() + .withDescription("Sort-TableScan(agg-pushed)") + .withOperandSupplier( + b0 -> + b0.operand(LogicalSort.class) + .predicate(hasOneFieldCollation.and(PlanUtils::sortByFieldsOnly)) + .oneInput( + b1 -> + b1.operand(CalciteLogicalIndexScan.class) + .predicate( + Predicate.not(AbstractCalciteIndexScan::noAggregatePushed)) + .noInputs())); + + @Override + default SortAggregateMeasureRule toRule() { + return new SortAggregateMeasureRule(this); + } + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/SortIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/SortIndexScanRule.java new file mode 100644 index 00000000000..ff30324d09f --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/SortIndexScanRule.java @@ -0,0 +1,68 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.planner.rules; + +import java.util.function.Predicate; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.core.Sort; +import org.immutables.value.Value; +import org.opensearch.sql.calcite.utils.PlanUtils; +import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan; + +@Value.Enclosing +public class SortIndexScanRule extends RelRule { + + protected SortIndexScanRule(Config config) { + super(config); + } + + @Override + public void onMatch(RelOptRuleCall call) { + final Sort sort = call.rel(0); + final AbstractCalciteIndexScan scan = call.rel(1); + if (sort.getConvention() != scan.getConvention()) { + return; + } + + var collations = sort.collation.getFieldCollations(); + AbstractCalciteIndexScan newScan = scan.pushDownSort(collations); + if (newScan != null) { + call.transformTo(newScan); + } + } + + /** Rule configuration. */ + @Value.Immutable + public interface Config extends RelRule.Config { + SortIndexScanRule.Config DEFAULT = + ImmutableSortIndexScanRule.Config.builder() + .build() + .withOperandSupplier( + b0 -> + b0.operand(Sort.class) + .predicate(PlanUtils::sortByFieldsOnly) + .oneInput( + b1 -> + b1.operand(AbstractCalciteIndexScan.class) + // Skip the rule if Top-K(i.e. sort + limit) has already been + // pushed down. Otherwise, + // Continue to push down sort although limit has already been + // pushed down since we don't promise collation with only limit. + .predicate( + Predicate.not(AbstractCalciteIndexScan::isTopKPushed) + .and( + Predicate.not( + AbstractCalciteIndexScan + ::isMetricsOrderPushed))) + .noInputs())); + + @Override + default SortIndexScanRule toRule() { + return new SortIndexScanRule(this); + } + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/SortProjectExprTransposeRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/SortProjectExprTransposeRule.java similarity index 96% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/SortProjectExprTransposeRule.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/SortProjectExprTransposeRule.java index dfec6754908..48684020909 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/SortProjectExprTransposeRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/SortProjectExprTransposeRule.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.opensearch.planner.physical; +package org.opensearch.sql.opensearch.planner.rules; import com.google.common.collect.ImmutableMap; import java.util.ArrayList; @@ -26,6 +26,7 @@ import org.apache.calcite.rex.RexNode; import org.apache.commons.lang3.tuple.Pair; import org.immutables.value.Value; +import org.opensearch.sql.calcite.utils.PlanUtils; import org.opensearch.sql.opensearch.util.OpenSearchRelOptUtil; /** @@ -132,7 +133,7 @@ public interface Config extends RelRule.Config { b1.operand(LogicalProject.class) .predicate( Predicate.not(LogicalProject::containsOver) - .and(OpenSearchIndexScanRule::projectContainsExpr)) + .and(PlanUtils::projectContainsExpr)) .anyInputs())); @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java index 065375dd230..b50201f3ea5 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java @@ -37,8 +37,11 @@ import com.google.common.collect.ImmutableList; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.function.Function; import lombok.RequiredArgsConstructor; import org.apache.calcite.plan.RelOptCluster; @@ -62,6 +65,7 @@ import org.opensearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder; import org.opensearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; import org.opensearch.search.aggregations.bucket.missing.MissingOrder; +import org.opensearch.search.aggregations.bucket.range.RangeAggregationBuilder; import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.opensearch.search.aggregations.metrics.ExtendedStats; import org.opensearch.search.aggregations.metrics.PercentilesAggregationBuilder; @@ -70,7 +74,6 @@ import org.opensearch.search.aggregations.support.ValueType; import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.opensearch.search.sort.SortOrder; -import org.opensearch.sql.ast.expression.Argument; import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.data.type.ExprCoreType; @@ -80,7 +83,6 @@ import org.opensearch.sql.opensearch.request.PredicateAnalyzer.NamedFieldExpression; import org.opensearch.sql.opensearch.response.agg.ArgMaxMinParser; import org.opensearch.sql.opensearch.response.agg.BucketAggregationParser; -import org.opensearch.sql.opensearch.response.agg.CompositeAggregationParser; import org.opensearch.sql.opensearch.response.agg.CountAsTotalHitsParser; import org.opensearch.sql.opensearch.response.agg.MetricParser; import org.opensearch.sql.opensearch.response.agg.NoBucketAggregationParser; @@ -89,7 +91,6 @@ import org.opensearch.sql.opensearch.response.agg.SingleValueParser; import org.opensearch.sql.opensearch.response.agg.StatsParser; import org.opensearch.sql.opensearch.response.agg.TopHitsParser; -import org.opensearch.sql.opensearch.storage.script.aggregation.dsl.BucketAggregationBuilder; import org.opensearch.sql.opensearch.storage.script.aggregation.dsl.CompositeAggregationBuilder; /** @@ -98,9 +99,6 @@ */ public class AggregateAnalyzer { - /** How many composite buckets should be returned. */ - public static final int AGGREGATION_BUCKET_SIZE = 1000; - /** metadata field used when there is no argument. Only apply to COUNT. */ private static final String METADATA_FIELD = "_index"; @@ -130,11 +128,12 @@ public static class ExpressionNotAnalyzableException extends Exception { private AggregateAnalyzer() {} @RequiredArgsConstructor - static class AggregateBuilderHelper { + public static class AggregateBuilderHelper { final RelDataType rowType; final Map fieldTypes; final RelOptCluster cluster; final boolean bucketNullable; + final int bucketSize; > T build(RexNode node, T aggBuilder) { return build(node, aggBuilder::field, aggBuilder::script); @@ -147,9 +146,14 @@ > T build(RexNode node, T sourceBuilde T build(RexNode node, Function fieldBuilder, Function scriptBuilder) { if (node == null) return fieldBuilder.apply(METADATA_FIELD); else if (node instanceof RexInputRef ref) { - return fieldBuilder.apply( - new NamedFieldExpression(ref.getIndex(), rowType.getFieldNames(), fieldTypes) - .getReferenceForTermQuery()); + // TODO : Workaround to ensure SQL plugin generates Composite aggs for Keyword field + // Else it performs a Query without any composite aggs to account for Null values. + NamedFieldExpression namedField = new NamedFieldExpression(ref.getIndex(), rowType.getFieldNames(), fieldTypes); + String fieldReference = namedField.getReferenceForTermQuery(); + if (fieldReference == null) { + fieldReference = namedField.getRootName(); + } + return fieldBuilder.apply(fieldReference); } else if (node instanceof RexCall || node instanceof RexLiteral) { return scriptBuilder.apply( (new PredicateAnalyzer.ScriptQueryExpression(node, rowType, fieldTypes, cluster)) @@ -179,101 +183,95 @@ T inferValue(RexNode node, Class clazz) { public static Pair, OpenSearchAggregationResponseParser> analyze( Aggregate aggregate, Project project, - RelDataType rowType, - Map fieldTypes, List outputFields, - RelOptCluster cluster) + AggregateBuilderHelper helper) throws ExpressionNotAnalyzableException { requireNonNull(aggregate, "aggregate"); try { - boolean bucketNullable = - Boolean.parseBoolean( - aggregate.getHints().stream() - .filter(hits -> hits.hintName.equals("stats_args")) - .map(hint -> hint.kvOptions.getOrDefault(Argument.BUCKET_NULLABLE, "true")) - .findFirst() - .orElseGet(() -> "true")); List groupList = aggregate.getGroupSet().asList(); - AggregateBuilderHelper helper = - new AggregateBuilderHelper(rowType, fieldTypes, cluster, bucketNullable); List aggFieldNames = outputFields.subList(groupList.size(), outputFields.size()); // Process all aggregate calls Pair> builderAndParser = processAggregateCalls(aggFieldNames, aggregate.getAggCallList(), project, helper); Builder metricBuilder = builderAndParser.getLeft(); - List metricParserList = builderAndParser.getRight(); + List metricParsers = builderAndParser.getRight(); // both count() and count(FIELD) can apply doc_count optimization in non-bucket aggregation, // but only count() can apply doc_count optimization in bucket aggregation. - boolean countAllOnly = !aggregate.getGroupSet().isEmpty(); - Pair, Builder> pair = - removeCountAggregationBuilders(metricBuilder, countAllOnly); - List removedCountAggBuilders = pair.getLeft(); - Builder newMetricBuilder = pair.getRight(); - - boolean removedCountAggBuildersHaveSomeField = - removedCountAggBuilders.stream() - .map(ValuesSourceAggregationBuilder::fieldName) - .distinct() - .count() - == 1; - boolean allCountAggRemoved = - removedCountAggBuilders.size() == metricBuilder.getAggregatorFactories().size(); +// FIXME : Put a check using Optimised index setting +// This is removed to ensure SQL plugin always sends a count() sub agg in the request instead of +// relying on the doc_count for the result. +// boolean countAllOnly = !groupList.isEmpty(); +// Pair, Builder> countAggNameAndBuilderPair = +// removeCountAggregationBuilders(metricBuilder, countAllOnly); + Builder newMetricBuilder = metricBuilder; + List countAggNames = Collections.emptyList(); + + // No group-by clause -- no parent aggregations are attached: + // - stats count() + // - stats avg(), count() + // Metric if (aggregate.getGroupSet().isEmpty()) { - if (allCountAggRemoved && removedCountAggBuildersHaveSomeField) { + if (newMetricBuilder == null) { // The optimization must require all count aggregations are removed, // and they have only one field name - List countAggNameList = - removedCountAggBuilders.stream() - .map(ValuesSourceAggregationBuilder::getName) - .toList(); - return Pair.of( - ImmutableList.copyOf(newMetricBuilder.getAggregatorFactories()), - new CountAsTotalHitsParser(countAggNameList)); + return Pair.of(List.of(), new CountAsTotalHitsParser(countAggNames)); } else { return Pair.of( - ImmutableList.copyOf(metricBuilder.getAggregatorFactories()), - new NoBucketAggregationParser(metricParserList)); + ImmutableList.copyOf(newMetricBuilder.getAggregatorFactories()), + new NoBucketAggregationParser(metricParsers)); } - } else if (aggregate.getGroupSet().length() == 1 - && isAutoDateSpan(project.getProjects().get(groupList.getFirst()))) { - RexCall rexCall = (RexCall) project.getProjects().get(groupList.getFirst()); - String bucketName = project.getRowType().getFieldList().get(groupList.getFirst()).getName(); - RexInputRef rexInputRef = (RexInputRef) rexCall.getOperands().getFirst(); - RexLiteral valueLiteral = (RexLiteral) rexCall.getOperands().get(1); - ValuesSourceAggregationBuilder bucketBuilder = - new AutoDateHistogramAggregationBuilder(bucketName) - .field(helper.inferNamedField(rexInputRef).getRootName()) - .setNumBuckets(requireNonNull(valueLiteral.getValueAs(Integer.class))); - return Pair.of( - Collections.singletonList(bucketBuilder.subAggregations(metricBuilder)), - new BucketAggregationParser(metricParserList)); } else { - List> buckets = - createCompositeBuckets(groupList, project, helper); - AggregationBuilder aggregationBuilder = - AggregationBuilders.composite("composite_buckets", buckets) - .size(AGGREGATION_BUCKET_SIZE); - - // For bucket aggregation, no count() aggregator or not all aggregators are count(), - // fallback to original ValueCountAggregation. - if (removedCountAggBuilders.isEmpty() - || removedCountAggBuilders.size() != metricBuilder.getAggregatorFactories().size()) { - aggregationBuilder.subAggregations(metricBuilder); + // Used to track the current sub-builder as analysis progresses + Builder subBuilder = newMetricBuilder; + // Push auto date span & case in group-by list into nested aggregations + Pair, AggregationBuilder> aggPushedAndAggBuilder = + createNestedAggregation(groupList, project, subBuilder, helper); + Set aggPushed = aggPushedAndAggBuilder.getLeft(); + AggregationBuilder pushedAggBuilder = aggPushedAndAggBuilder.getRight(); + // The group-by list after removing pushed aggregations + groupList = groupList.stream().filter(i -> !aggPushed.contains(i)).toList(); + if (pushedAggBuilder != null) { + subBuilder = new Builder().addAggregator(pushedAggBuilder); + } + + // No composite aggregation at top-level -- auto date span & case in group-by list are + // pushed into nested aggregations: + // - stats avg() by range_field + // - stats count() by auto_date_span + // - stats count() by ...auto_date_spans, ...range_fields + // [AutoDateHistogram | RangeAgg]+ + // Metric + if (groupList.isEmpty()) { return Pair.of( - Collections.singletonList(aggregationBuilder), - new CompositeAggregationParser(metricParserList)); + ImmutableList.copyOf(subBuilder.getAggregatorFactories()), + new BucketAggregationParser(metricParsers, countAggNames)); } - // No need to register sub-factories if no aggregator factories left after removing all - // ValueCountAggregationBuilder. - if (!newMetricBuilder.getAggregatorFactories().isEmpty()) { - aggregationBuilder.subAggregations(newMetricBuilder); + // Composite aggregation at top level -- it has composite aggregation, with or without its + // incompatible value sources as sub-aggregations: + // - stats avg() by term_fields + // - stats avg() by date_histogram + // - stats count() by auto_date_span, range_field, term_fields + // CompositeAgg + // [AutoDateHistogram | RangeAgg]* + // Metric + else { + List> buckets = + createCompositeBuckets(groupList, project, helper); + if (buckets.size() != groupList.size()) { + throw new UnsupportedOperationException( + "Not all the left aggregations can be converted to value sources of composite" + + " aggregation"); + } + AggregationBuilder compositeBuilder = + AggregationBuilders.composite("composite_buckets", buckets).size(helper.bucketSize); + if (subBuilder != null) { + compositeBuilder.subAggregations(subBuilder); + } + return Pair.of( + Collections.singletonList(compositeBuilder), + new BucketAggregationParser(metricParsers, countAggNames)); } - List countAggNameList = - removedCountAggBuilders.stream().map(ValuesSourceAggregationBuilder::getName).toList(); - return Pair.of( - Collections.singletonList(aggregationBuilder), - new CompositeAggregationParser(metricParserList, countAggNameList)); } } catch (Throwable e) { Throwables.throwIfInstanceOf(e, UnsupportedOperationException.class); @@ -282,14 +280,16 @@ && isAutoDateSpan(project.getProjects().get(groupList.getFirst()))) { } /** - * Remove all ValueCountAggregationBuilder from metric builder, and return the removed - * ValueCountAggregationBuilder list. + * Remove all ValueCountAggregationBuilder from metric builder, and return the name list for the + * removed count aggs with the updated metric builder. * * @param metricBuilder metrics builder * @param countAllOnly remove count() only, or count(FIELD) will be removed. - * @return a pair of removed ValueCountAggregationBuilder and updated metric builder + * @return a pair of name list for the removed count aggs and updated metric builder. If the count + * aggregations cannot satisfy the requirement to remove, it will return an empty name list + * with the original metric builder. */ - private static Pair, Builder> removeCountAggregationBuilders( + private static Pair, Builder> removeCountAggregationBuilders( Builder metricBuilder, boolean countAllOnly) { List countAggregatorFactories = metricBuilder.getAggregatorFactories().stream() @@ -302,7 +302,26 @@ private static Pair, Builder> removeCountAggr copy.removeAll(countAggregatorFactories); Builder newMetricBuilder = new AggregatorFactories.Builder(); copy.forEach(newMetricBuilder::addAggregator); - return Pair.of(countAggregatorFactories, newMetricBuilder); + + if (countAllOnly || supportCountFiled(countAggregatorFactories, metricBuilder)) { + List countAggNameList = + countAggregatorFactories.stream().map(ValuesSourceAggregationBuilder::getName).toList(); + if (newMetricBuilder.getAggregatorFactories().isEmpty()) { + newMetricBuilder = null; + } + return Pair.of(countAggNameList, newMetricBuilder); + } + return Pair.of(List.of(), metricBuilder); + } + + private static boolean supportCountFiled( + List countAggBuilderList, Builder metricBuilder) { + return countAggBuilderList.size() == metricBuilder.getAggregatorFactories().size() + && countAggBuilderList.stream() + .map(ValuesSourceAggregationBuilder::fieldName) + .distinct() + .count() + == 1; } private static Pair> processAggregateCalls( @@ -339,7 +358,7 @@ private static Pair createAggregationBuilderAn AggregateCall aggCall, List args, String aggFieldName, - AggregateBuilderHelper helper) { + AggregateAnalyzer.AggregateBuilderHelper helper) { if (aggCall.isDistinct()) { return createDistinctAggregation(aggCall, args, aggFieldName, helper); } else { @@ -374,6 +393,8 @@ private static Pair createRegularAggregation( case AVG -> Pair.of( helper.build(args.getFirst(), AggregationBuilders.avg(aggFieldName)), new SingleValueParser(aggFieldName)); + // 1. Only case SUM, skip SUM0 / COUNT since calling avg() in DSL should be faster. + // 2. To align with databases, SUM0 is not preferred now. case SUM -> Pair.of( helper.build(args.getFirst(), AggregationBuilders.sum(aggFieldName)), new SingleValueParser(aggFieldName)); @@ -382,9 +403,8 @@ private static Pair createRegularAggregation( !args.isEmpty() ? args.getFirst() : null, AggregationBuilders.count(aggFieldName)), new SingleValueParser(aggFieldName)); case MIN -> { - String fieldName = helper.inferNamedField(args.getFirst()).getRootName(); - ExprType fieldType = helper.fieldTypes.get(fieldName); - + ExprType fieldType = + OpenSearchTypeFactory.convertRelDataTypeToExprType(args.getFirst().getType()); if (supportsMaxMinAggregation(fieldType)) { yield Pair.of( helper.build(args.getFirst(), AggregationBuilders.min(aggFieldName)), @@ -392,7 +412,7 @@ private static Pair createRegularAggregation( } else { yield Pair.of( AggregationBuilders.topHits(aggFieldName) - .fetchSource(helper.inferNamedField(args.getFirst()).getRootName(), null) + .fetchField(helper.inferNamedField(args.getFirst()).getReferenceForTermQuery()) .size(1) .from(0) .sort( @@ -402,9 +422,8 @@ private static Pair createRegularAggregation( } } case MAX -> { - String fieldName = helper.inferNamedField(args.getFirst()).getRootName(); - ExprType fieldType = helper.fieldTypes.get(fieldName); - + ExprType fieldType = + OpenSearchTypeFactory.convertRelDataTypeToExprType(args.getFirst().getType()); if (supportsMaxMinAggregation(fieldType)) { yield Pair.of( helper.build(args.getFirst(), AggregationBuilders.max(aggFieldName)), @@ -412,7 +431,7 @@ private static Pair createRegularAggregation( } else { yield Pair.of( AggregationBuilders.topHits(aggFieldName) - .fetchSource(helper.inferNamedField(args.getFirst()).getRootName(), null) + .fetchField(helper.inferNamedField(args.getFirst()).getReferenceForTermQuery()) .size(1) .from(0) .sort( @@ -435,20 +454,20 @@ private static Pair createRegularAggregation( new StatsParser(ExtendedStats::getStdDeviationPopulation, aggFieldName)); case ARG_MAX -> Pair.of( AggregationBuilders.topHits(aggFieldName) - .fetchSource(helper.inferNamedField(args.getFirst()).getRootName(), null) + .fetchField(helper.inferNamedField(args.getFirst()).getReferenceForTermQuery()) .size(1) .from(0) .sort( - helper.inferNamedField(args.get(1)).getRootName(), + helper.inferNamedField(args.get(1)).getReferenceForTermQuery(), org.opensearch.search.sort.SortOrder.DESC), new ArgMaxMinParser(aggFieldName)); case ARG_MIN -> Pair.of( AggregationBuilders.topHits(aggFieldName) - .fetchSource(helper.inferNamedField(args.getFirst()).getRootName(), null) + .fetchField(helper.inferNamedField(args.getFirst()).getReferenceForTermQuery()) .size(1) .from(0) .sort( - helper.inferNamedField(args.get(1)).getRootName(), + helper.inferNamedField(args.get(1)).getReferenceForTermQuery(), org.opensearch.search.sort.SortOrder.ASC), new ArgMaxMinParser(aggFieldName)); case OTHER_FUNCTION -> { @@ -457,7 +476,7 @@ private static Pair createRegularAggregation( yield switch (functionName) { case TAKE -> Pair.of( AggregationBuilders.topHits(aggFieldName) - .fetchSource(helper.inferNamedField(args.getFirst()).getRootName(), null) + .fetchField(helper.inferNamedField(args.getFirst()).getReferenceForTermQuery()) .size(helper.inferValue(args.getLast(), Integer.class)) .from(0), new TopHitsParser(aggFieldName)); @@ -465,7 +484,8 @@ yield switch (functionName) { TopHitsAggregationBuilder firstBuilder = AggregationBuilders.topHits(aggFieldName).size(1).from(0); if (!args.isEmpty()) { - firstBuilder.fetchSource(helper.inferNamedField(args.getFirst()).getRootName(), null); + firstBuilder.fetchField( + helper.inferNamedField(args.getFirst()).getReferenceForTermQuery()); } yield Pair.of(firstBuilder, new TopHitsParser(aggFieldName, true)); } @@ -476,7 +496,8 @@ yield switch (functionName) { .from(0) .sort("_doc", org.opensearch.search.sort.SortOrder.DESC); if (!args.isEmpty()) { - lastBuilder.fetchSource(helper.inferNamedField(args.getFirst()).getRootName(), null); + lastBuilder.fetchField( + helper.inferNamedField(args.getFirst()).getReferenceForTermQuery()); } yield Pair.of(lastBuilder, new TopHitsParser(aggFieldName, true)); } @@ -491,6 +512,11 @@ yield switch (functionName) { } yield Pair.of(aggBuilder, new SinglePercentileParser(aggFieldName)); } + case DISTINCT_COUNT_APPROX -> Pair.of( + helper.build( + !args.isEmpty() ? args.getFirst() : null, + AggregationBuilders.cardinality(aggFieldName)), + new SingleValueParser(aggFieldName)); default -> throw new AggregateAnalyzer.AggregateAnalyzerException( String.format("Unsupported push-down aggregator %s", aggCall.getAggregation())); }; @@ -512,11 +538,6 @@ private static boolean supportsMaxMinAggregation(ExprType fieldType) { || coreType == ExprCoreType.TIMESTAMP; } - private static ValuesSourceAggregationBuilder createBucketAggregation( - Integer group, Project project, AggregateAnalyzer.AggregateBuilderHelper helper) { - return createBucket(group, project, helper); - } - private static List> createCompositeBuckets( List groupList, Project project, AggregateAnalyzer.AggregateBuilderHelper helper) { ImmutableList.Builder> resultBuilder = ImmutableList.builder(); @@ -525,37 +546,122 @@ private static List> createCompositeBuckets( return resultBuilder.build(); } + /** + * Creates nested bucket aggregations for expressions that are not qualified as value sources for + * composite aggregations. + * + *

This method processes a list of group by expressions and identifies those that cannot be + * used as value sources in composite aggregations but can be pushed down as sub-aggregations, + * such as auto date histograms and range buckets. + * + *

The aggregation hierarchy follows this pattern: + * + *

+   * AutoDateHistogram | RangeAggregation
+   *   └── AutoDateHistogram | RangeAggregation (nested)
+   *       └── ... (more composite-incompatible aggregations)
+   *           └── Metric Aggregation (at the bottom)
+   * 
+ * + * @param groupList the list of group by field indices from the query + * @param project the projection containing the expressions to analyze + * @param metricBuilder the metric aggregation builder to be placed at the bottom of the hierarchy + * @param helper the aggregation builder helper containing row type and utility methods + * @return a pair containing: + *
    + *
  • A set of integers representing the indices of group fields that were successfully + * pushed as sub-aggregations + *
  • The root aggregation builder, or null if no such expressions were found + *
+ */ + private static Pair, AggregationBuilder> createNestedAggregation( + List groupList, + Project project, + Builder metricBuilder, + AggregateAnalyzer.AggregateBuilderHelper helper) { + AggregationBuilder rootAggBuilder = null; + AggregationBuilder tailAggBuilder = null; + + Set aggPushed = new HashSet<>(); + for (Integer i : groupList) { + RexNode agg = project.getProjects().get(i); + String name = project.getNamedProjects().get(i).getValue(); + AggregationBuilder aggBuilder = createCompositeIncompatibleAggregation(agg, name, helper); + if (aggBuilder != null) { + aggPushed.add(i); + if (rootAggBuilder == null) { + rootAggBuilder = aggBuilder; + } else { + tailAggBuilder.subAggregation(aggBuilder); + } + tailAggBuilder = aggBuilder; + } + } + if (tailAggBuilder != null && metricBuilder != null) { + tailAggBuilder.subAggregations(metricBuilder); + } + return Pair.of(aggPushed, rootAggBuilder); + } + + /** + * Creates an aggregation builder for expressions that are not qualified as composite aggregation + * value sources. + * + *

This method analyzes RexNode expressions and creates appropriate OpenSearch aggregation + * builders for cases where they can not be value sources of a composite aggregation. + * + *

The method supports the following aggregation types: + * + *

+   * - Auto Date Histogram Aggregation: For temporal bucketing with automatic interval selection
+   * - Range Aggregation: For CASE expressions that define value ranges
+   * 
+ * + * @param agg the RexNode expression to analyze and convert + * @param name the name to assign to the created aggregation builder + * @param helper the aggregation builder helper containing row type and utility methods + * @return the appropriate ValuesSourceAggregationBuilder for the expression, or null if no + * compatible aggregation type is found + */ + private static ValuesSourceAggregationBuilder createCompositeIncompatibleAggregation( + RexNode agg, String name, AggregateBuilderHelper helper) { + ValuesSourceAggregationBuilder aggBuilder = null; + if (isAutoDateSpan(agg)) { + aggBuilder = analyzeAutoDateSpan(agg, name, helper); + } else if (isCase(agg)) { + Optional rangeAggBuilder = + CaseRangeAnalyzer.create(name, helper.rowType).analyze((RexCall) agg); + if (rangeAggBuilder.isPresent()) { + aggBuilder = rangeAggBuilder.get(); + } + } + return aggBuilder; + } + + private static AutoDateHistogramAggregationBuilder analyzeAutoDateSpan( + RexNode spanAgg, String name, AggregateAnalyzer.AggregateBuilderHelper helper) { + RexCall rexCall = (RexCall) spanAgg; + RexInputRef rexInputRef = (RexInputRef) rexCall.getOperands().getFirst(); + RexLiteral valueLiteral = (RexLiteral) rexCall.getOperands().get(1); + return new AutoDateHistogramAggregationBuilder(name) + .field(helper.inferNamedField(rexInputRef).getRootName()) + .setNumBuckets(requireNonNull(valueLiteral.getValueAs(Integer.class))); + } + private static boolean isAutoDateSpan(RexNode rex) { return rex instanceof RexCall rexCall && rexCall.getKind() == SqlKind.OTHER_FUNCTION && rexCall.getOperator().equals(WIDTH_BUCKET); } - private static ValuesSourceAggregationBuilder createBucket( - Integer groupIndex, Project project, AggregateBuilderHelper helper) { - RexNode rex = project.getProjects().get(groupIndex); - String bucketName = project.getRowType().getFieldList().get(groupIndex).getName(); - if (rex instanceof RexCall rexCall - && rexCall.getKind() == SqlKind.OTHER_FUNCTION - && rexCall.getOperator().getName().equalsIgnoreCase(BuiltinFunctionName.SPAN.name()) - && rexCall.getOperands().size() == 3 - && rexCall.getOperands().getFirst() instanceof RexInputRef rexInputRef - && rexCall.getOperands().get(1) instanceof RexLiteral valueLiteral - && rexCall.getOperands().get(2) instanceof RexLiteral unitLiteral) { - return BucketAggregationBuilder.buildHistogram( - bucketName, - helper.inferNamedField(rexInputRef).getRootName(), - valueLiteral.getValueAs(Double.class), - SpanUnit.of(unitLiteral.getValueAs(String.class))); - } else { - return createTermsAggregationBuilder(bucketName, rex, helper); - } + private static boolean isCase(RexNode rex) { + return rex instanceof RexCall rexCall && rexCall.getKind() == SqlKind.CASE; } private static CompositeValuesSourceBuilder createCompositeBucket( Integer groupIndex, Project project, AggregateAnalyzer.AggregateBuilderHelper helper) { RexNode rex = project.getProjects().get(groupIndex); - String bucketName = project.getRowType().getFieldList().get(groupIndex).getName(); + String bucketName = project.getRowType().getFieldNames().get(groupIndex); if (rex instanceof RexCall rexCall && rexCall.getKind() == SqlKind.OTHER_FUNCTION && rexCall.getOperator().getName().equalsIgnoreCase(BuiltinFunctionName.SPAN.name()) @@ -570,10 +676,6 @@ private static CompositeValuesSourceBuilder createCompositeBucket( SpanUnit.of(unitLiteral.getValueAs(String.class)), MissingOrder.FIRST, helper.bucketNullable); - } else if (isAutoDateSpan(rex)) { - // Defense check. We've already prevented this case in OpenSearchAggregateIndexScanRule. - throw new UnsupportedOperationException( - "auto_date_histogram is not supported in composite agg."); } else { return createTermsSourceBuilder(bucketName, rex, helper); } @@ -588,13 +690,11 @@ private static CompositeValuesSourceBuilder createTermsSourceBuilder( } CompositeValuesSourceBuilder sourceBuilder = helper.build(group, termsBuilder); - // Time types values are converted to LONG in ExpressionAggregationScript::execute - if (List.of(TIMESTAMP, TIME, DATE) - .contains(OpenSearchTypeFactory.convertRelDataTypeToExprType(group.getType()))) { - sourceBuilder.userValuetypeHint(ValueType.LONG); - } - - return sourceBuilder; + return withValueTypeHint( + sourceBuilder, + sourceBuilder::userValuetypeHint, + group.getType(), + group instanceof RexInputRef); } private static ValuesSourceAggregationBuilder createTermsAggregationBuilder( @@ -603,14 +703,33 @@ private static ValuesSourceAggregationBuilder createTermsAggregationBuilder( helper.build( group, new TermsAggregationBuilder(bucketName) - .size(AGGREGATION_BUCKET_SIZE) + .size(helper.bucketSize) .order(BucketOrder.key(true))); + return withValueTypeHint( + sourceBuilder, + sourceBuilder::userValueTypeHint, + group.getType(), + group instanceof RexInputRef); + } + + private static T withValueTypeHint( + T sourceBuilder, + Function withValueTypeHint, + RelDataType groupType, + boolean isSourceField) { + ExprType exprType = OpenSearchTypeFactory.convertRelDataTypeToExprType(groupType); // Time types values are converted to LONG in ExpressionAggregationScript::execute - if (List.of(TIMESTAMP, TIME, DATE) - .contains(OpenSearchTypeFactory.convertRelDataTypeToExprType(group.getType()))) { - sourceBuilder.userValueTypeHint(ValueType.LONG); + if (List.of(TIMESTAMP, TIME, DATE).contains(exprType)) { + return withValueTypeHint.apply(ValueType.LONG); } - - return sourceBuilder; + // No need to set type hints for source fields + if (isSourceField) { + return sourceBuilder; + } + ValueType valueType = ValueType.lenientParse(exprType.typeName().toLowerCase()); + // The default value type is STRING, don't set that explicitly to avoid plan change. + return valueType == null || valueType == ValueType.STRING + ? sourceBuilder + : withValueTypeHint.apply(valueType); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/CaseRangeAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/CaseRangeAnalyzer.java new file mode 100644 index 00000000000..104ab04e547 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/CaseRangeAnalyzer.java @@ -0,0 +1,289 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.request; + +import com.google.common.collect.BoundType; +import com.google.common.collect.Range; +import com.google.common.collect.RangeSet; +import com.google.common.collect.TreeRangeSet; +import java.math.BigDecimal; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlBinaryOperator; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.util.Sarg; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.bucket.range.RangeAggregationBuilder; + +/** + * Analyzer to detect CASE expressions that can be converted to OpenSearch range aggregations. + * + *

Strict validation rules: + * + *

    + *
  • All conditions must compare the same field with literals + *
  • Only closed-open, at-least, and less-than ranges are allowed + *
  • Return values must be string literals + *
+ */ +public class CaseRangeAnalyzer { + /** The default key to use if there isn't a key specified for the else case */ + public static final String DEFAULT_ELSE_KEY = "null"; + + private final RelDataType rowType; + private final RangeSet takenRange; + private final RangeAggregationBuilder builder; + + public CaseRangeAnalyzer(String name, RelDataType rowType) { + this.rowType = rowType; + this.takenRange = TreeRangeSet.create(); + this.builder = AggregationBuilders.range(name).keyed(true); + } + + /** + * Creates a new CaseRangeAnalyzer instance. + * + * @param rowType the row type information for field resolution + * @return a new CaseRangeAnalyzer instance + */ + public static CaseRangeAnalyzer create(String name, RelDataType rowType) { + return new CaseRangeAnalyzer(name, rowType); + } + + /** + * Analyzes a CASE expression to determine if it can be converted to a range aggregation. + * + * @param caseCall The CASE RexCall to analyze + * @return Optional RangeAggregationBuilder if conversion is possible, empty otherwise + */ + public Optional analyze(RexCall caseCall) { + if (!caseCall.getKind().equals(SqlKind.CASE)) { + return Optional.empty(); + } + + List operands = caseCall.getOperands(); + + // Process WHEN-THEN pairs + for (int i = 0; i < operands.size() - 1; i += 2) { + RexNode condition = operands.get(i); + RexNode expr = operands.get(i + 1); + try { + String key = parseLiteralAsString(expr); + analyzeCondition(condition, key); + } catch (UnsupportedOperationException e) { + return Optional.empty(); + } + } + + // Check ELSE clause + RexNode elseExpr = operands.getLast(); + String elseKey; + if (RexLiteral.isNullLiteral(elseExpr)) { + // range key doesn't support values of type: VALUE_NULL + elseKey = DEFAULT_ELSE_KEY; + } else { + try { + elseKey = parseLiteralAsString(elseExpr); + } catch (UnsupportedOperationException e) { + return Optional.empty(); + } + } + addRangeSet(elseKey, takenRange.complement()); + return Optional.of(builder); + } + + /** Analyzes a single condition in the CASE WHEN clause. */ + private void analyzeCondition(RexNode condition, String key) { + if (!(condition instanceof RexCall)) { + throwUnsupported("condition must be a RexCall"); + } + + RexCall call = (RexCall) condition; + SqlKind kind = call.getKind(); + + // Handle simple comparisons + if (kind == SqlKind.GREATER_THAN_OR_EQUAL + || kind == SqlKind.LESS_THAN + || kind == SqlKind.LESS_THAN_OR_EQUAL + || kind == SqlKind.GREATER_THAN) { + analyzeSimpleComparison(call, key); + } else if (kind == SqlKind.SEARCH) { + analyzeSearchCondition(call, key); + } + // AND / OR will only appear when users try to create a complex condition on multiple fields + // E.g. (a > 3 and b < 5). Otherwise, the complex conditions will be converted to a SEARCH call. + else if (kind == SqlKind.AND || kind == SqlKind.OR) { + throwUnsupported("Range queries must be performed on the same field"); + } else { + throwUnsupported("Can not analyze condition as a range query: " + call); + } + } + + private void analyzeSimpleComparison(RexCall call, String key) { + List operands = call.getOperands(); + if (operands.size() != 2 || !(call.getOperator() instanceof SqlBinaryOperator)) { + throwUnsupported(); + } + RexNode left = operands.get(0); + RexNode right = operands.get(1); + SqlOperator operator = call.getOperator(); + RexInputRef inputRef = null; + RexLiteral literal = null; + + // Swap inputRef to the left if necessary + if (left instanceof RexInputRef && right instanceof RexLiteral) { + inputRef = (RexInputRef) left; + literal = (RexLiteral) right; + } else if (left instanceof RexLiteral && right instanceof RexInputRef) { + inputRef = (RexInputRef) right; + literal = (RexLiteral) left; + operator = operator.reverse(); + } else { + throwUnsupported(); + } + + if (operator == null) { + throwUnsupported(); + } + + String fieldName = rowType.getFieldNames().get(inputRef.getIndex()); + if (builder.field() == null) { + builder.field(fieldName); + } else if (!Objects.equals(builder.field(), fieldName)) { + throwUnsupported("comparison must be performed on the same field"); + } + + Double value = literal.getValueAs(Double.class); + if (value == null) { + throwUnsupported("Cannot parse value for comparison"); + } + switch (operator.getKind()) { + case GREATER_THAN_OR_EQUAL -> { + addFrom(key, value); + } + case LESS_THAN -> { + addTo(key, value); + } + default -> throw new UnsupportedOperationException( + "ranges must be equivalents of field >= constant or field < constant"); + } + ; + } + + private void analyzeSearchCondition(RexCall searchCall, String key) { + RexNode field = searchCall.getOperands().getFirst(); + if (!(field instanceof RexInputRef)) { + throwUnsupported("Range query must be performed on a field"); + } + String fieldName = getFieldName((RexInputRef) field); + if (builder.field() == null) { + builder.field(fieldName); + } else if (!Objects.equals(builder.field(), fieldName)) { + throwUnsupported("Range query must be performed on the same field"); + } + RexLiteral literal = (RexLiteral) searchCall.getOperands().getLast(); + Sarg sarg = Objects.requireNonNull(literal.getValueAs(Sarg.class)); + for (Range r : sarg.rangeSet.asRanges()) { + @SuppressWarnings("unchecked") + Range range = (Range) r; + validateRange(range); + if (!range.hasLowerBound() && range.hasUpperBound()) { + // It will be Double.MAX_VALUE if be big decimal is greater than Double.MAX_VALUE + double upper = range.upperEndpoint().doubleValue(); + addTo(key, upper); + } else if (range.hasLowerBound() && !range.hasUpperBound()) { + double lower = range.lowerEndpoint().doubleValue(); + addFrom(key, lower); + } else if (range.hasLowerBound()) { + double lower = range.lowerEndpoint().doubleValue(); + double upper = range.upperEndpoint().doubleValue(); + addBetween(key, lower, upper); + } else { + addBetween(key, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); + } + } + } + + private void addFrom(String key, Double value) { + var from = Range.atLeast(value); + updateRange(key, from); + } + + private void addTo(String key, Double value) { + var to = Range.lessThan(value); + updateRange(key, to); + } + + private void addBetween(String key, Double from, Double to) { + var range = Range.closedOpen(from, to); + updateRange(key, range); + } + + private void updateRange(String key, Range range) { + // The range to add: remaining space ∩ new range + RangeSet toAdd = takenRange.complement().subRangeSet(range); + addRangeSet(key, toAdd); + takenRange.add(range); + } + + // Add range set without updating taken range + private void addRangeSet(String key, RangeSet rangeSet) { + rangeSet.asRanges().forEach(range -> addRange(key, range)); + } + + // Add range without updating taken range + private void addRange(String key, Range range) { + validateRange(range); + if (range.hasLowerBound() && range.hasUpperBound()) { + builder.addRange(key, range.lowerEndpoint(), range.upperEndpoint()); + } else if (range.hasLowerBound()) { + builder.addUnboundedFrom(key, range.lowerEndpoint()); + } else if (range.hasUpperBound()) { + builder.addUnboundedTo(key, range.upperEndpoint()); + } else { + builder.addRange(key, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); + } + } + + private String getFieldName(RexInputRef field) { + return rowType.getFieldNames().get(field.getIndex()); + } + + private static void validateRange(Range range) { + if ((range.hasLowerBound() && range.lowerBoundType() != BoundType.CLOSED) + || (range.hasUpperBound() && range.upperBoundType() != BoundType.OPEN)) { + throwUnsupported("Range query only supports closed-open ranges"); + } + } + + private static String parseLiteralAsString(RexNode node) { + if (!(node instanceof RexLiteral)) { + throwUnsupported("Result expressions of range queries must be literals"); + } + RexLiteral literal = (RexLiteral) node; + try { + return literal.getValueAs(String.class); + } catch (AssertionError ignore) { + } + throw new UnsupportedOperationException( + "Cannot parse result expression of type " + literal.getType()); + } + + private static void throwUnsupported() { + throw new UnsupportedOperationException("Cannot create range aggregator from case"); + } + + private static void throwUnsupported(String message) { + throw new UnsupportedOperationException("Cannot create range aggregator: " + message); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java index c6a3e6197db..d5ad7bcf416 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java @@ -5,20 +5,57 @@ package org.opensearch.sql.opensearch.request; -import static org.opensearch.core.xcontent.DeprecationHandler.IGNORE_DEPRECATIONS; -import static org.opensearch.search.sort.FieldSortBuilder.DOC_FIELD_NAME; -import static org.opensearch.search.sort.SortOrder.ASC; -import static org.opensearch.sql.opensearch.storage.OpenSearchIndex.METADATA_FIELD_ID; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.function.Consumer; -import java.util.function.Function; +import io.substrait.extension.DefaultExtensionCatalog; +import io.substrait.extension.SimpleExtension; +import io.substrait.isthmus.ImmutableFeatureBoard; +import io.substrait.isthmus.SubstraitRelVisitor; +import io.substrait.isthmus.TypeConverter; +import io.substrait.isthmus.UserTypeMapper; +import io.substrait.isthmus.expression.AggregateFunctionConverter; +import io.substrait.isthmus.expression.FunctionMappings; +import io.substrait.isthmus.expression.ScalarFunctionConverter; +import io.substrait.isthmus.expression.WindowFunctionConverter; +import io.substrait.plan.Plan; +import io.substrait.plan.PlanProtoConverter; +import io.substrait.relation.NamedScan; +import io.substrait.relation.Rel; +import io.substrait.relation.RelCopyOnWriteVisitor; +import io.substrait.type.Type; +import io.substrait.type.TypeCreator; +import io.substrait.util.EmptyVisitationContext; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; -import org.opensearch.action.search.*; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.rel.RelShuttleImpl; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.logical.LogicalFilter; +import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlLibraryOperators; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.tools.FrameworkConfig; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.util.Pair; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.search.SearchScrollRequest; +import org.opensearch.common.Nullable; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; @@ -31,10 +68,38 @@ import org.opensearch.search.SearchModule; import org.opensearch.search.builder.PointInTimeBuilder; import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.sort.FieldSortBuilder; +import org.opensearch.sql.calcite.plan.LogicalSystemLimit; +import org.opensearch.sql.calcite.type.ExprSqlType; +import org.opensearch.sql.calcite.utils.CalciteToolsHelper; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.executor.OpenSearchTypeSystem; +import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.PPLBuiltinOperators; +import org.opensearch.sql.expression.function.udf.datetime.ExtractFunction; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.response.OpenSearchResponse; import org.opensearch.sql.opensearch.storage.OpenSearchIndex; import org.opensearch.sql.opensearch.storage.OpenSearchStorageEngine; +import org.opensearch.sql.opensearch.storage.scan.CalciteLogicalIndexScan; + +import java.io.IOException; +import java.sql.Connection; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.apache.calcite.sql.fun.SqlLibraryOperators.REGEXP_REPLACE_3; +import static org.apache.calcite.sql.fun.SqlLibraryOperators.SAFE_CAST; +import static org.opensearch.core.xcontent.DeprecationHandler.IGNORE_DEPRECATIONS; +import static org.opensearch.search.sort.FieldSortBuilder.DOC_FIELD_NAME; +import static org.opensearch.search.sort.SortOrder.ASC; /** * OpenSearch search request. This has to be stateful because it needs to: @@ -47,11 +112,24 @@ @ToString public class OpenSearchQueryRequest implements OpenSearchRequest { + private static final SimpleExtension.ExtensionCollection EXTENSIONS; + + static { + try { + SimpleExtension.ExtensionCollection customExtension = SimpleExtension.load(List.of("/opensearch_custom_functions.yaml")); + EXTENSIONS = DefaultExtensionCatalog.DEFAULT_COLLECTION.merge(customExtension); + } catch (Exception e) { + throw new RuntimeException("Failed to load custom extensions", e); + } + } + + public static final String INJECTED_COUNT_AGGREGATE_NAME = "agg_for_doc_count"; + /** {@link OpenSearchRequest.IndexName}. */ private final IndexName indexName; /** Search request source builder. */ - private SearchSourceBuilder sourceBuilder; + private final SearchSourceBuilder sourceBuilder; /** OpenSearchExprValueFactory. */ @EqualsAndHashCode.Exclude @ToString.Exclude @@ -73,6 +151,9 @@ public class OpenSearchQueryRequest implements OpenSearchRequest { private SearchResponse searchResponse = null; + private static final Logger LOGGER = + LogManager.getLogger(OpenSearchQueryRequest.class); + /** Constructor of OpenSearchQueryRequest. */ public OpenSearchQueryRequest( String indexName, int size, OpenSearchExprValueFactory factory, List includes) { @@ -177,6 +258,7 @@ public OpenSearchResponse search( // get the value before set searchDone = true boolean isCountAggRequest = isCountAggRequest(); searchDone = true; + sourceBuilder.queryPlanIR(convertToSubstraitAndSerialize(exprValueFactory)); return new OpenSearchResponse( searchAction.apply( new SearchRequest().indices(indexName.getIndexNames()).source(sourceBuilder)), @@ -198,17 +280,29 @@ public OpenSearchResponse searchWithPIT(Function SearchHits.empty(), exprValueFactory, includes, isCountAggRequest()); } else { this.sourceBuilder.pointInTimeBuilder(new PointInTimeBuilder(this.pitId)); + sourceBuilder.queryPlanIR(convertToSubstraitAndSerialize(exprValueFactory)); this.sourceBuilder.timeout(cursorKeepAlive); // check for search after if (searchAfter != null) { this.sourceBuilder.searchAfter(searchAfter); } - // Set sort field for search_after - if (this.sourceBuilder.sorts() == null) { + // Add sort tiebreaker for PIT search. + // We cannot remove it since `_shard_doc` is not added implicitly in PIT now. + // Ref https://github.com/opensearch-project/OpenSearch/pull/18924#issuecomment-3342365950 + if (this.sourceBuilder.sorts() == null || this.sourceBuilder.sorts().isEmpty()) { + // If no sort field specified, sort by `_doc` + `_shard_doc`to get better performance this.sourceBuilder.sort(DOC_FIELD_NAME, ASC); - // Workaround to preserve sort location more exactly, - // see https://github.com/opensearch-project/sql/pull/3061 - this.sourceBuilder.sort(METADATA_FIELD_ID, ASC); +// this.sourceBuilder.sort(SortBuilders.shardDocSort()); + } else { + // If sort fields specified, sort by `fields` + `_doc` + `_shard_doc`. + if (this.sourceBuilder.sorts().stream() + .noneMatch( + b -> b instanceof FieldSortBuilder f && f.fieldName().equals(DOC_FIELD_NAME))) { + this.sourceBuilder.sort(DOC_FIELD_NAME, ASC); + } +// if (this.sourceBuilder.sorts().stream().noneMatch(ShardDocSortBuilder.class::isInstance)) { +// this.sourceBuilder.sort(SortBuilders.shardDocSort()); +// } } SearchRequest searchRequest = new SearchRequest().indices(indexName.getIndexNames()).source(this.sourceBuilder); @@ -286,4 +380,527 @@ public void writeTo(StreamOutput out) throws IOException { "OpenSearchQueryRequest serialization is not implemented."); } } + + public static byte[] convertToSubstraitAndSerialize(OpenSearchExprValueFactory index) { + RelNode relNode = CalciteToolsHelper.OpenSearchRelRunners.getCurrentRelNode(); + CalciteToolsHelper.OpenSearchRelRunners.clearCurrentRelNode(); + LOGGER.info("Calcite Logical Plan before Conversion\n {}", RelOptUtil.toString(relNode)); + + // Preprocess the Calcite plan + relNode = preprocessRelNodes(relNode); + // Adds a count aggregate if absent for Coordinator merging using doc_count to work + relNode = ensureCountAggregate(relNode); + // Support to convert average into sum and count aggs else merging at Coordinator won't work. + relNode = convertAvgToSumCount(relNode); + // Support to convert COUNT(DISTINCT) to APPROX_COUNT_DISTINCT for partial results + relNode = convertCountDistinctToApprox(relNode); + + LOGGER.info("Calcite Logical Plan after Conversion\n {}", RelOptUtil.toString(relNode)); + + long startTimeSubstrait = System.nanoTime(); + // Substrait conversion + // RelRoot represents the root of a relational query tree with metadata + RelRoot root = RelRoot.of(relNode, SqlKind.SELECT); + + // Convert using custom visitor to handle EXTRACT and other custom functions + SubstraitRelVisitor visitor = createVisitor(relNode); + Rel substraitRel = visitor.apply(root.rel); + + // Build Plan.Root with proper field names from RelRoot + // otherwise the output column names won't match the query + List fieldNames = root.fields.stream() + .map(field -> field.getValue()) + .collect(Collectors.toList()); + Plan.Root substraitRoot = Plan.Root.builder() + .input(substraitRel) + .names(fieldNames) + .build(); + + long endTimeSubstraitConvert = System.nanoTime(); + // Plan contains one or more roots (query entry points) and shared extensions + // addRoots() adds the converted relation tree as a query root + Plan plan = Plan.builder().addRoots(substraitRoot).build(); + // The Plan now contains two table names like bellow + // named_table { + // names: "OpenSearch" + // names: "hits" + // } + // we want to remove "OpenSearch" as table name for now otherwise execution fails in DF + TableNameModifier modifier = new TableNameModifier(); + Plan modifiedPlan = modifier.modifyTableNames(plan); + + // Convert to Protocol Buffer format for serialization + // PlanProtoConverter handles the conversion from Java objects to protobuf + // This enables serialization, storage, and cross-system communication + PlanProtoConverter planProtoConverter = new PlanProtoConverter(); + io.substrait.proto.Plan substraitPlanProtoModified = planProtoConverter.toProto(modifiedPlan); + LOGGER.info("Time taken to convert to Substrait convert (ms) {}", (endTimeSubstraitConvert-startTimeSubstrait)/1000000); + LOGGER.info("Substrait Logical Plan \n {}", substraitPlanProtoModified.toString()); + return substraitPlanProtoModified.toByteArray(); + } + + private static SubstraitRelVisitor createVisitor(RelNode relNode) { + //Mapping of Function names in Calcite to Substrait + List customSigs = List.of( + new FunctionMappings.Sig(PPLBuiltinOperators.EXTRACT, "date_part"), + new FunctionMappings.Sig(PPLBuiltinOperators.STRFTIME, "date_format"), + new FunctionMappings.Sig(PPLBuiltinOperators.DATE_FORMAT, "date_format"), + new FunctionMappings.Sig(REGEXP_REPLACE_3, "regexp_replace"), + new FunctionMappings.Sig(SqlLibraryOperators.ILIKE, "like") + ); + + TypeConverter typeConverter = new TypeConverter( + new UserTypeMapper() { + @Nullable + @Override + public Type toSubstrait(RelDataType relDataType) { + if(isTimeStampUDT(relDataType)) { + TypeCreator creator = Type.withNullability(relDataType.isNullable()); + return creator.precisionTimestamp(3); + } + return null; + } + + @Nullable + @Override + public RelDataType toCalcite(Type.UserDefined type) { + return null; + } + }); + + RelDataTypeFactory typeFactory = relNode.getCluster().getTypeFactory(); + AggregateFunctionConverter aggConverter = new AggregateFunctionConverter( + EXTENSIONS.aggregateFunctions(), + typeFactory + ); + ScalarFunctionConverter scalarConverter = new ScalarFunctionConverter( + EXTENSIONS.scalarFunctions(), + customSigs, + typeFactory, + typeConverter + ); + WindowFunctionConverter windowConverter = new WindowFunctionConverter( + EXTENSIONS.windowFunctions(), + typeFactory + ); + + return new SubstraitRelVisitor( + typeFactory, + scalarConverter, + aggConverter, + windowConverter, + typeConverter, + ImmutableFeatureBoard.builder().build()); + } + + private static class TableNameModifier { + public Plan modifyTableNames(Plan plan) { + TableNameVisitor visitor = new TableNameVisitor(); + + // Transform each root in the plan + List modifiedRoots = new java.util.ArrayList<>(); + + for (Plan.Root root : plan.getRoots()) { + Optional modifiedRel = root.getInput().accept(visitor, null); + if (modifiedRel.isPresent()) { + modifiedRoots.add(Plan.Root.builder().from(root).input(modifiedRel.get()).build()); + } else { + modifiedRoots.add(root); + } + } + + return Plan.builder().from(plan).roots(modifiedRoots).build(); + } + + private static class TableNameVisitor extends RelCopyOnWriteVisitor { + @Override + public Optional visit(NamedScan namedScan, EmptyVisitationContext context) { + List currentNames = namedScan.getNames(); + + // Filter out names that contain "OpenSearch" + List filteredNames = currentNames.stream() + .filter(name -> !name.contains("OpenSearch")) + .collect(java.util.stream.Collectors.toList()); + + // Only create a new NamedScan if names were actually filtered + if (filteredNames.size() != currentNames.size() && !filteredNames.isEmpty()) { + return Optional.of( + NamedScan.builder() + .from(namedScan) + .names(filteredNames) + .build()); + } + + return super.visit(namedScan, context); + } + } + } + + private static RelNode convertAvgToSumCount(RelNode relNode) { + // Track: original AVG field index → (new SUM index, new COUNT index) + Map> avgFieldMapping = new HashMap<>(); + // Track: original field index → new field index (for non-AVG fields) + Map fieldIndexMapping = new HashMap<>(); + + return relNode.accept( + new RelShuttleImpl() { + + @Override + public RelNode visit(LogicalAggregate aggregate) { + RelNode newInput = aggregate.getInput().accept(this); + + boolean hasAvg = + aggregate.getAggCallList().stream() + .anyMatch(call -> call.getAggregation().getKind() == SqlKind.AVG); + + if (!hasAvg) { + return aggregate.copy(aggregate.getTraitSet(), Collections.singletonList(newInput)); + } + + FrameworkConfig config = Frameworks.newConfigBuilder() + .typeSystem(OpenSearchTypeSystem.INSTANCE) + .build(); + Connection connection = CalciteToolsHelper.connect(config, OpenSearchTypeFactory.TYPE_FACTORY); + RelBuilder builder = CalciteToolsHelper.create(config, OpenSearchTypeFactory.TYPE_FACTORY, connection); + builder.push(newInput); + + List newAggCalls = new ArrayList<>(); + int newFieldIndex = aggregate.getGroupCount(); + + // First, map group fields (they don't change) + for (int i = 0; i < aggregate.getGroupCount(); i++) { + fieldIndexMapping.put(i, i); + } + + for (int i = 0; i < aggregate.getAggCallList().size(); i++) { + AggregateCall aggCall = aggregate.getAggCallList().get(i); + int originalFieldIndex = aggregate.getGroupCount() + i; + + if (aggCall.getAggregation().getKind() == SqlKind.AVG) { + avgFieldMapping.put(originalFieldIndex, Pair.of(newFieldIndex, newFieldIndex + 1)); + + newAggCalls.add( + builder.sum( + aggCall.isDistinct(), + aggCall.getName() + "_sum", + builder.field(aggCall.getArgList().get(0)))); + newAggCalls.add( + builder.count( + aggCall.isDistinct(), + aggCall.getName() + "_count", + builder.field(aggCall.getArgList().get(0)))); + newFieldIndex += 2; // avg is adding 2 fields: sum & count + } else { + fieldIndexMapping.put(originalFieldIndex, newFieldIndex); + newAggCalls.add( + builder + .aggregateCall( + aggCall.getAggregation(), + aggCall.getArgList().stream() + .map(builder::field) + .collect(Collectors.toList())) + .distinct(aggCall.isDistinct()) + .as(aggCall.getName())); + newFieldIndex++; + } + } + + builder.aggregate( + builder.groupKey(aggregate.getGroupSet(), aggregate.getGroupSets()), newAggCalls); + return builder.build(); + } + + @Override + public RelNode visit(LogicalProject project) { + RelNode newInput = project.getInput().accept(this); + + if (avgFieldMapping.isEmpty()) { + return project.copy(project.getTraitSet(), Collections.singletonList(newInput)); + } + + FrameworkConfig config = Frameworks.newConfigBuilder() + .typeSystem(OpenSearchTypeSystem.INSTANCE) + .build(); + Connection connection = CalciteToolsHelper.connect(config, OpenSearchTypeFactory.TYPE_FACTORY); + RelBuilder builder = CalciteToolsHelper.create(config, OpenSearchTypeFactory.TYPE_FACTORY, connection); + builder.push(newInput); + + List newProjects = new ArrayList<>(); + List newNames = new ArrayList<>(); + + for (int i = 0; i < project.getProjects().size(); i++) { + RexNode expr = project.getProjects().get(i); + String name = project.getRowType().getFieldNames().get(i); + + if (expr instanceof RexInputRef) { + RexInputRef inputRef = (RexInputRef) expr; + + // Check if this is an AVG field that needs expansion + Pair avgMapping = avgFieldMapping.get(inputRef.getIndex()); + if (avgMapping != null) { + // Add both SUM and COUNT columns + newProjects.add(builder.field(avgMapping.left)); + newNames.add(name + "_sum"); + newProjects.add(builder.field(avgMapping.right)); + newNames.add(name + "_count"); + continue; + } + + // Otherwise, remap the field index for non-AVG fields + Integer newIndex = fieldIndexMapping.get(inputRef.getIndex()); + if (newIndex != null) { + newProjects.add(builder.field(newIndex)); + newNames.add(name); + continue; + } + } + + // Keep other expressions as-is + newProjects.add(expr); + newNames.add(name); + } + + builder.project(newProjects, newNames); + return builder.build(); + } + }); + } + + private static RelNode convertCountDistinctToApprox(RelNode relNode) { + return relNode.accept(new RelShuttleImpl() { + @Override + public RelNode visit(LogicalAggregate aggregate) { + RelNode newInput = aggregate.getInput().accept(this); + + boolean hasCountDistinct = aggregate.getAggCallList().stream() + .anyMatch(call -> call.getAggregation().getKind() == SqlKind.COUNT && call.isDistinct()); + + if (!hasCountDistinct) { + return aggregate.copy(aggregate.getTraitSet(), Collections.singletonList(newInput)); + } + + List newAggCalls = new ArrayList<>(); + for (AggregateCall aggCall : aggregate.getAggCallList()) { + if (aggCall.getAggregation().getKind() == SqlKind.COUNT && aggCall.isDistinct()) { + // Replace COUNT(DISTINCT x) with APPROX_COUNT_DISTINCT(x) + AggregateCall newCall = AggregateCall.create( + SqlStdOperatorTable.APPROX_COUNT_DISTINCT , + false, // not distinct anymore since APPROX_COUNT_DISTINCT handles it + aggCall.isApproximate(), + aggCall.getArgList(), + aggCall.filterArg, + aggCall.collation, + aggCall.getType(), + aggCall.getName() + ); + newAggCalls.add(newCall); + } else { + newAggCalls.add(aggCall); + } + } + + return LogicalAggregate.create( + newInput, + aggregate.getHints(), + aggregate.getGroupSet(), + aggregate.getGroupSets(), + newAggCalls + ); + } + }); + } + + private static boolean isTimeStampUDT(RelDataType relDataType) { + if (relDataType.getClass().equals(ExprSqlType.class)) { + ExprSqlType exprSqlType = (ExprSqlType) relDataType; + return exprSqlType.getUdt().equals(OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP); + } + return false; + } + + private static boolean isSpanFunction(RexCall rexCall) { + return rexCall.getKind() == SqlKind.OTHER_FUNCTION + && rexCall.getOperator().getName().equalsIgnoreCase(BuiltinFunctionName.SPAN.name()); + } + + private static boolean isILikeFunction(RexCall rexCall) { + return rexCall.getOperator() == SqlLibraryOperators.ILIKE; + } + + private static RexNode updateUDF(RexNode rexNode, RexBuilder rexBuilder) { + // Handle SPAN Function + if (rexNode instanceof RexCall rexCall && isSpanFunction(rexCall)) { + List operands = rexCall.getOperands(); + // SPAN(field, divisor, unit) -> FLOOR(field / divisor) * divisor + if (operands.size() >= 2) { + RexNode field = operands.get(0); + RexNode divisor = operands.get(1); + RexNode division = rexBuilder.makeCall(SqlStdOperatorTable.DIVIDE, field, divisor); + RexNode realDivision = rexBuilder.makeCast( + rexBuilder.getTypeFactory().createSqlType(SqlTypeName.REAL), + division); + RexNode floor = rexBuilder.makeCall(SqlStdOperatorTable.FLOOR, realDivision); + return rexBuilder.makeCall(SqlStdOperatorTable.MULTIPLY, floor, divisor); + } + } + + // Handle ILIKE Function + if (rexNode instanceof RexCall rexCall && isILikeFunction(rexCall)) { + List operands = rexCall.getOperands(); + // We need exactly 2 operands if we want to match this with substrait like function + if (operands.size() >= 2) { + RexNode field = operands.get(0); + RexNode pattern = operands.get(1); + RexNode upperField = rexBuilder.makeCall(SqlStdOperatorTable.UPPER, field); + RexNode upperPattern = rexBuilder.makeCall(SqlStdOperatorTable.UPPER, pattern); + return rexBuilder.makeCall(rexCall.getOperator(), upperField, upperPattern); + } + } + + // Handle TimeStamp Function + if(rexNode instanceof RexCall rexCall) { + List originalOperands = rexCall.getOperands(); + List updatedOperands = new ArrayList<>(); + for (RexNode operand : originalOperands) { + if(operand instanceof RexCall timestampCall && isTimeStampUDT(timestampCall.getType())) { + org.apache.calcite.rel.type.RelDataType timestampType = rexBuilder.getTypeFactory().createSqlType(SqlTypeName.TIMESTAMP, 3); + updatedOperands.add(rexBuilder.makeCall(timestampCall.pos, timestampType, SAFE_CAST, timestampCall.getOperands())); + } else if(operand instanceof RexInputRef timeStampInput && isTimeStampUDT(timeStampInput.getType())) { + org.apache.calcite.rel.type.RelDataType timestampType = rexBuilder.getTypeFactory().createSqlType(SqlTypeName.TIMESTAMP, 3); + updatedOperands.add(rexBuilder.makeInputRef(timestampType, timeStampInput.getIndex())); + } else { + updatedOperands.add(updateUDF(operand, rexBuilder)); + } + } + return rexBuilder.makeCall(rexCall.pos, rexCall.type, rexCall.getOperator(), updatedOperands); + } + return rexNode; + } + + private static RelNode ensureCountAggregate(RelNode relNode) { + return relNode.accept( + new RelShuttleImpl() { + @Override + public RelNode visit(LogicalAggregate aggregate) { + boolean hasCount = + aggregate.getAggCallList().stream() + .anyMatch(call -> call.getAggregation().getKind() == SqlKind.COUNT); + + if (hasCount) { + return super.visit(aggregate); + } + + FrameworkConfig config = Frameworks.newConfigBuilder() + .typeSystem(OpenSearchTypeSystem.INSTANCE) + .build(); + Connection connection = CalciteToolsHelper.connect(config, OpenSearchTypeFactory.TYPE_FACTORY); + RelBuilder builder = CalciteToolsHelper.create(config, OpenSearchTypeFactory.TYPE_FACTORY, connection); + builder.push(aggregate.getInput()); + + List aggCalls = new ArrayList<>(); + for (AggregateCall call : aggregate.getAggCallList()) { + aggCalls.add( + builder + .aggregateCall( + call.getAggregation(), + call.getArgList().stream() + .map(builder::field) + .collect(Collectors.toList())) + .distinct(call.isDistinct()) + .as(call.getName())); + } + aggCalls.add(builder.count(false, INJECTED_COUNT_AGGREGATE_NAME)); + + builder.aggregate(builder.groupKey(aggregate.getGroupSet()), aggCalls); + return builder.build(); + } + + @Override + public RelNode visit(LogicalProject project) { + RelNode input = project.getInput().accept(this); + if (input == project.getInput()) { + return project; + } + + if (input instanceof LogicalAggregate agg) { + int countIndex = agg.getGroupCount() + agg.getAggCallList().size() - 1; + RexBuilder rexBuilder = project.getCluster().getRexBuilder(); + + List newProjects = new ArrayList<>(project.getProjects()); + newProjects.add( + rexBuilder.makeInputRef( + agg.getRowType().getFieldList().get(countIndex).getType(), countIndex)); + + List newNames = new ArrayList<>(project.getRowType().getFieldNames()); + newNames.add(INJECTED_COUNT_AGGREGATE_NAME); + + return LogicalProject.create(input, project.getHints(), newProjects, newNames); + } + + return project; + } + }); + } + + private static RelNode preprocessRelNodes(RelNode relNode) { + return relNode.accept(new RelShuttleImpl() { + + @Override + public RelNode visit(LogicalProject logicalProject) { + RelNode newInput = logicalProject.getInput().accept(this); + List updatedProjects = logicalProject.getProjects().stream() + .map(project -> updateUDF(project, logicalProject.getCluster().getRexBuilder())) + .collect(Collectors.toList()); + + return LogicalProject.create( + newInput, + logicalProject.getHints(), + updatedProjects, + logicalProject.getRowType() + ); + } + + @Override + public RelNode visit(LogicalFilter logicalFilter) { + RelNode newInput = logicalFilter.getInput().accept(this); + RexNode originalCondition = logicalFilter.getCondition(); + RexNode updatedCondition = updateUDF(originalCondition, logicalFilter.getCluster().getRexBuilder()); + return LogicalFilter.create(newInput, updatedCondition); + } + + @Override + public RelNode visit(LogicalAggregate logicalAggregate) { + RelNode newInput = logicalAggregate.getInput().accept(this); + return LogicalAggregate.create( + newInput, + logicalAggregate.getHints(), + logicalAggregate.getGroupSet(), + logicalAggregate.getGroupSets(), + logicalAggregate.getAggCallList() + ); + } + + @Override + public RelNode visit(TableScan tableScan) { + return tableScan; + } + + @Override + public RelNode visit(RelNode other) { + if (other instanceof LogicalSystemLimit) { + LogicalSystemLimit limit = (LogicalSystemLimit) other; + RelNode newInput = limit.getInput().accept(this); + return LogicalSystemLimit.create( + limit.getType(), + newInput, + limit.getCollation(), + limit.offset, + limit.fetch + ); + } + return super.visit(other); + } + }); + } + } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java index 5f9cdf8a5d9..59051ca6ef0 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java @@ -1274,11 +1274,9 @@ public QueryExpression notLike(LiteralExpression literal) { @Override public QueryExpression equals(LiteralExpression literal) { Object value = literal.value(); - if (value instanceof GregorianCalendar) { + if (literal.isDateTime()) { builder = - boolQuery() - .must(addFormatIfNecessary(literal, rangeQuery(getFieldReference()).gte(value))) - .must(addFormatIfNecessary(literal, rangeQuery(getFieldReference()).lte(value))); + addFormatIfNecessary(literal, rangeQuery(getFieldReference()).gte(value).lte(value)); } else { builder = termQuery(getFieldReferenceForTermQuery(), value); } @@ -1288,7 +1286,7 @@ public QueryExpression equals(LiteralExpression literal) { @Override public QueryExpression notEquals(LiteralExpression literal) { Object value = literal.value(); - if (value instanceof GregorianCalendar) { + if (literal.isDateTime()) { builder = boolQuery() .should(addFormatIfNecessary(literal, rangeQuery(getFieldReference()).gt(value))) @@ -1399,8 +1397,12 @@ public QueryExpression notIn(LiteralExpression literal) { @Override public QueryExpression equals(Object point, boolean isTimeStamp) { - builder = - termQuery(getFieldReferenceForTermQuery(), convertEndpointValue(point, isTimeStamp)); + Object value = convertEndpointValue(point, isTimeStamp); + if (isTimeStamp) { + builder = rangeQuery(getFieldReference()).gte(value).lte(value).format("date_time"); + } else { + builder = termQuery(getFieldReferenceForTermQuery(), value); + } return this; } @@ -1419,6 +1421,7 @@ public QueryExpression between(Range range, boolean isTimeStamp) { range.upperBoundType() == BoundType.CLOSED ? rangeQueryBuilder.lte(upperBound) : rangeQueryBuilder.lt(upperBound); + if (isTimeStamp) rangeQueryBuilder.format("date_time"); builder = rangeQueryBuilder; return this; } @@ -1511,7 +1514,7 @@ public List getUnAnalyzableNodes() { */ private static RangeQueryBuilder addFormatIfNecessary( LiteralExpression literal, RangeQueryBuilder rangeQueryBuilder) { - if (literal.value() instanceof GregorianCalendar) { + if (literal.isDateTime()) { rangeQueryBuilder.format("date_time"); } return rangeQueryBuilder; @@ -1634,7 +1637,7 @@ Object value() { return doubleValue(); } else if (isBoolean()) { return booleanValue(); - } else if (isTimestamp()) { + } else if (isTimestamp() || isDate()) { return timestampValueForPushDown(RexLiteral.stringValue(literal)); } else if (isString()) { return RexLiteral.stringValue(literal); @@ -1676,10 +1679,21 @@ public boolean isTimestamp() { return false; } + public boolean isDate() { + if (literal.getType() instanceof ExprSqlType exprSqlType) { + return exprSqlType.getUdt() == ExprUDT.EXPR_DATE; + } + return false; + } + public boolean isIp() { return literal.getType() instanceof ExprIPType; } + public boolean isDateTime() { + return rawValue() instanceof GregorianCalendar || isTimestamp() || isDate(); + } + long longValue() { return ((Number) literal.getValue()).longValue(); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/AbstractBucketAggregationParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/AbstractBucketAggregationParser.java new file mode 100644 index 00000000000..1098ccd4666 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/AbstractBucketAggregationParser.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.response.agg; + +import java.util.List; +import java.util.Map; +import org.opensearch.search.SearchHits; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregation; +import org.opensearch.search.aggregations.bucket.range.Range; + +/** + * Abstract base class for parsing bucket aggregations from OpenSearch responses. Provides common + * functionality for extracting key-value pairs from different types of buckets. + */ +public abstract class AbstractBucketAggregationParser + implements OpenSearchAggregationResponseParser { + /** + * Extracts key-value pairs from a composite aggregation bucket without processing its + * sub-aggregations. + * + *

For example, for the following CompositeAggregation bucket in response: + * + *

{@code
+   * {
+   *   "key": {
+   *     "firstname": "William",
+   *     "lastname": "Shakespeare"
+   *   },
+   *   "sub_agg_name": {
+   *     "buckets": []
+   *   }
+   * }
+   * }
+ * + * It returns {@code {"firstname": "William", "lastname": "Shakespeare"}} as the response. + * + * @param bucket the composite aggregation bucket to extract data from + * @return a map containing the bucket's key-value pairs + */ + protected Map extract(CompositeAggregation.Bucket bucket) { + return bucket.getKey(); + } + + /** + * Extracts key-value pairs from a range aggregation bucket without processing its + * sub-aggregations. + * + * @param bucket the range aggregation bucket to extract data from + * @param name the name to use as the key in the returned map + * @return a map containing the bucket's key mapped to the provided name + */ + protected Map extract(Range.Bucket bucket, String name) { + return Map.of(name, bucket.getKey()); + } + + @Override + public List> parse(SearchHits hits) { + throw new UnsupportedOperationException(this.getClass() + " doesn't support parse(SearchHits)"); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/ArgMaxMinParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/ArgMaxMinParser.java index 2ff1e511713..55dacd7081c 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/ArgMaxMinParser.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/ArgMaxMinParser.java @@ -27,13 +27,12 @@ public Map parse(Aggregation agg) { return Collections.singletonMap(agg.getName(), null); } - Map source = hits[0].getSourceAsMap(); - - if (source.isEmpty()) { - return Collections.singletonMap(agg.getName(), null); - } else { - Object value = source.values().iterator().next(); + // Get value from fields (fetchField) + if (hits[0].getFields() != null && !hits[0].getFields().isEmpty()) { + Object value = hits[0].getFields().values().iterator().next().getValue(); return Collections.singletonMap(agg.getName(), value); } + + return Collections.singletonMap(agg.getName(), null); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/BucketAggregationParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/BucketAggregationParser.java index f43743d2e28..db6e4eef248 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/BucketAggregationParser.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/BucketAggregationParser.java @@ -6,23 +6,32 @@ package org.opensearch.sql.opensearch.response.agg; import java.util.Arrays; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.stream.Collectors; +import java.util.stream.IntStream; import lombok.EqualsAndHashCode; +import lombok.Getter; import org.opensearch.search.SearchHits; import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.Aggregations; import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregation; +import org.opensearch.search.aggregations.bucket.histogram.InternalAutoDateHistogram; +import org.opensearch.search.aggregations.bucket.range.Range; +import org.opensearch.search.aggregations.bucket.terms.InternalMultiTerms; /** - * Use BucketAggregationParser only when there is a single group-by key, it returns multiple - * buckets. {@link CompositeAggregationParser} is used for multiple group by keys + * Use BucketAggregationParser for {@link MultiBucketsAggregation}, where it returns multiple + * buckets. */ @EqualsAndHashCode public class BucketAggregationParser implements OpenSearchAggregationResponseParser { - private final MetricParserHelper metricsParser; + @Getter private final MetricParserHelper metricsParser; + // countAggNameList dedicated the list of count aggregations which are filled by doc_count + private List countAggNameList = List.of(); public BucketAggregationParser(MetricParser... metricParserList) { metricsParser = new MetricParserHelper(Arrays.asList(metricParserList)); @@ -32,18 +41,52 @@ public BucketAggregationParser(List metricParserList) { metricsParser = new MetricParserHelper(metricParserList); } + public BucketAggregationParser( + List metricParserList, List countAggNameList) { + metricsParser = new MetricParserHelper(metricParserList, countAggNameList); + this.countAggNameList = countAggNameList; + } + @Override public List> parse(Aggregations aggregations) { Aggregation agg = aggregations.asList().getFirst(); return ((MultiBucketsAggregation) agg) - .getBuckets().stream().map(b -> parse(b, agg.getName())).collect(Collectors.toList()); + .getBuckets().stream() + .map(b -> parseBucket(b, agg.getName())) + .filter(Objects::nonNull) + .flatMap(List::stream) + .toList(); + } + + private List> parseBucket( + MultiBucketsAggregation.Bucket bucket, String name) { + // return null so that an empty bucket of range or date span will be filtered out + if (bucket instanceof Range.Bucket || bucket instanceof InternalAutoDateHistogram.Bucket) { + if (bucket.getDocCount() == 0) { + return null; + } + } + + Aggregations aggregations = bucket.getAggregations(); + List> results = + isLeafAgg(aggregations) + ? parseLeafAgg(aggregations, bucket.getDocCount()) + : parse(aggregations); + + Optional> common = extract(bucket, name); + common.ifPresent(commonMap -> results.forEach(r -> r.putAll(commonMap))); + return results; + } + + private boolean isLeafAgg(Aggregations aggregations) { + return !(aggregations.asList().size() == 1 + && aggregations.asList().get(0) instanceof MultiBucketsAggregation); } - private Map parse(MultiBucketsAggregation.Bucket bucket, String keyName) { - Map resultMap = new LinkedHashMap<>(); - resultMap.put(keyName, bucket.getKey()); - resultMap.putAll(metricsParser.parse(bucket.getAggregations())); - return resultMap; + private List> parseLeafAgg(Aggregations aggregations, long docCount) { + Map resultMap = metricsParser.parse(aggregations); + countAggNameList.forEach(countAggName -> resultMap.put(countAggName, docCount)); + return List.of(resultMap); } @Override @@ -51,4 +94,49 @@ public List> parse(SearchHits hits) { throw new UnsupportedOperationException( "BucketAggregationParser doesn't support parse(SearchHits)"); } + + /** + * Extracts key-value pairs from different types of aggregation buckets without processing their + * sub-aggregations. + * + *

For CompositeAggregation buckets, it extracts all key-value pairs from the bucket's key. For + * example, for the following CompositeAggregation bucket in response: + * + *

{@code
+   * {
+   *   "key": {
+   *     "firstname": "William",
+   *     "lastname": "Shakespeare"
+   *   },
+   *   "sub_agg_name": {
+   *     "buckets": []
+   *   }
+   * }
+   * }
+ * + * It returns {@code {"firstname": "William", "lastname": "Shakespeare"}}. + * + *

For Range buckets, it creates a single key-value pair using the provided name and the + * bucket's key. + * + * @param bucket the aggregation bucket to extract data from + * @param name the aggregation name + * @return an Optional containing the extracted key-value pairs + */ + protected Optional> extract( + MultiBucketsAggregation.Bucket bucket, String name) { + Map extracted; + if (bucket instanceof CompositeAggregation.Bucket compositeBucket) { + extracted = compositeBucket.getKey(); + } else if (bucket instanceof InternalMultiTerms.Bucket) { + List keys = Arrays.asList(name.split("\\|")); + extracted = + IntStream.range(0, keys.size()) + .boxed() + .collect(Collectors.toMap(keys::get, ((List) bucket.getKey())::get)); + } else { + extracted = Map.of(name, bucket.getKey()); + } + return Optional.ofNullable(extracted); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/MetricParserHelper.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/MetricParserHelper.java index 8886668abb0..efe61adb8a5 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/MetricParserHelper.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/MetricParserHelper.java @@ -15,6 +15,7 @@ import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.Aggregations; import org.opensearch.sql.common.utils.StringUtils; +import org.opensearch.sql.opensearch.request.OpenSearchQueryRequest; /** Parse multiple metrics in one bucket. */ @Getter diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/TopHitsParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/TopHitsParser.java index 9179670d1a7..c9d9cf61d9e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/TopHitsParser.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/TopHitsParser.java @@ -42,21 +42,24 @@ public Map parse(Aggregation agg) { } if (returnSingleValue) { - // Extract the single value from the first (and only) hit - Map source = hits[0].getSourceAsMap(); - if (source.isEmpty()) { - return Collections.singletonMap(agg.getName(), null); + // Extract the single value from the first (and only) hit from fields (fetchField) + if (hits[0].getFields() != null && !hits[0].getFields().isEmpty()) { + Object value = hits[0].getFields().values().iterator().next().getValue(); + return Collections.singletonMap(agg.getName(), value); } - // Get the first value from the source map - Object value = source.values().iterator().next(); - return Collections.singletonMap(agg.getName(), value); + return Collections.singletonMap(agg.getName(), null); } else { - // Return all values as a list - return Collections.singletonMap( - agg.getName(), - Arrays.stream(hits) - .flatMap(h -> h.getSourceAsMap().values().stream()) - .collect(Collectors.toList())); + // Return all values as a list from fields (fetchField) + if (hits[0].getFields() != null && !hits[0].getFields().isEmpty()) { + return Collections.singletonMap( + agg.getName(), + Arrays.stream(hits) + .flatMap(h -> h.getFields().values().stream()) + .map(f -> f.getValue()) + .filter(v -> v != null) // Filter out null values + .collect(Collectors.toList())); + } + return Collections.singletonMap(agg.getName(), Collections.emptyList()); } } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index d1bc05c3895..9141c5a1837 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -28,6 +28,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.TimeValue; import org.opensearch.index.IndexSettings; +import org.opensearch.search.aggregations.MultiBucketConsumerService; import org.opensearch.sql.common.setting.Settings; /** Setting implementation on OpenSearch. */ @@ -125,7 +126,21 @@ public class OpenSearchSettings extends Settings { Setting.intSetting( Key.PPL_VALUES_MAX_LIMIT.getKeyValue(), 0, - 0, + -1, + Setting.Property.NodeScope, + Setting.Property.Dynamic); + + public static final Setting PPL_SUBSEARCH_MAXOUT_SETTING = + Setting.intSetting( + Key.PPL_SUBSEARCH_MAXOUT.getKeyValue(), + 10000, + Setting.Property.NodeScope, + Setting.Property.Dynamic); + + public static final Setting PPL_JOIN_SUBSEARCH_MAXOUT_SETTING = + Setting.intSetting( + Key.PPL_JOIN_SUBSEARCH_MAXOUT.getKeyValue(), + 50000, Setting.Property.NodeScope, Setting.Property.Dynamic); @@ -171,7 +186,7 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); - public static final Setting QUERY_SIZE_LIMIT_SETTING = + public static final Setting QUERY_SIZE_LIMIT_SETTING = Setting.intSetting( Key.QUERY_SIZE_LIMIT.getKeyValue(), IndexSettings.MAX_RESULT_WINDOW_SETTING, @@ -179,6 +194,15 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); + // Set the default value to QUERY_SIZE_LIMIT_SETTING + public static final Setting QUERY_BUCKET_SIZE_SETTING = + Setting.intSetting( + Key.QUERY_BUCKET_SIZE.getKeyValue(), + OpenSearchSettings.QUERY_SIZE_LIMIT_SETTING, + 0, + Setting.Property.NodeScope, + Setting.Property.Dynamic); + public static final Setting METRICS_ROLLING_WINDOW_SETTING = Setting.longSetting( Key.METRICS_ROLLING_WINDOW.getKeyValue(), @@ -388,6 +412,18 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.PPL_VALUES_MAX_LIMIT, PPL_VALUES_MAX_LIMIT_SETTING, new Updater(Key.PPL_VALUES_MAX_LIMIT)); + register( + settingBuilder, + clusterSettings, + Key.PPL_SUBSEARCH_MAXOUT, + PPL_SUBSEARCH_MAXOUT_SETTING, + new Updater(Key.PPL_SUBSEARCH_MAXOUT)); + register( + settingBuilder, + clusterSettings, + Key.PPL_JOIN_SUBSEARCH_MAXOUT, + PPL_JOIN_SUBSEARCH_MAXOUT_SETTING, + new Updater(Key.PPL_JOIN_SUBSEARCH_MAXOUT)); register( settingBuilder, clusterSettings, @@ -430,6 +466,18 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.QUERY_SIZE_LIMIT, QUERY_SIZE_LIMIT_SETTING, new Updater(Key.QUERY_SIZE_LIMIT)); + register( + settingBuilder, + clusterSettings, + Key.QUERY_BUCKET_SIZE, + QUERY_BUCKET_SIZE_SETTING, + new Updater(Key.QUERY_BUCKET_SIZE)); + register( + settingBuilder, + clusterSettings, + Key.SEARCH_MAX_BUCKETS, + MultiBucketConsumerService.MAX_BUCKET_SETTING, + new Updater(Key.SEARCH_MAX_BUCKETS)); register( settingBuilder, clusterSettings, @@ -603,8 +651,11 @@ public static List> pluginSettings() { .add(DEFAULT_PATTERN_SHOW_NUMBERED_TOKEN_SETTING) .add(PPL_REX_MAX_MATCH_LIMIT_SETTING) .add(PPL_VALUES_MAX_LIMIT_SETTING) + .add(PPL_SUBSEARCH_MAXOUT_SETTING) + .add(PPL_JOIN_SUBSEARCH_MAXOUT_SETTING) .add(QUERY_MEMORY_LIMIT_SETTING) .add(QUERY_SIZE_LIMIT_SETTING) + .add(QUERY_BUCKET_SIZE_SETTING) .add(METRICS_ROLLING_WINDOW_SETTING) .add(METRICS_ROLLING_INTERVAL_SETTING) .add(DATASOURCE_URI_HOSTS_DENY_LIST) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java index c8b00c6daed..ddb27328bbb 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java @@ -17,6 +17,7 @@ import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.util.CompositeMap; import org.opensearch.common.unit.TimeValue; import org.opensearch.sql.calcite.plan.AbstractOpenSearchTable; import org.opensearch.sql.common.setting.Settings; @@ -152,6 +153,11 @@ public Map getReservedFieldTypes() { return METADATAFIELD_TYPE_MAP; } + // Return all field types including reserved fields + public Map getAllFieldTypes() { + return CompositeMap.of(getFieldTypes(), getReservedFieldTypes()); + } + public Map getAliasMapping() { if (cachedFieldOpenSearchTypes == null) { cachedFieldOpenSearchTypes = @@ -159,7 +165,7 @@ public Map getAliasMapping() { } if (aliasMapping == null) { aliasMapping = - cachedFieldOpenSearchTypes.entrySet().stream() + OpenSearchDataType.traverseAndFlatten(cachedFieldOpenSearchTypes).entrySet().stream() .filter(entry -> entry.getValue().getOriginalPath().isPresent()) .collect( Collectors.toUnmodifiableMap( @@ -190,6 +196,12 @@ public Integer getMaxResultWindow() { return cachedMaxResultWindow; } + public Integer getBucketSize() { + return Math.min( + settings.getSettingValue(Settings.Key.QUERY_BUCKET_SIZE), + settings.getSettingValue(Settings.Key.SEARCH_MAX_BUCKETS)); + } + /** TODO: Push down operations to index scan operator as much as possible in future. */ @Override public PhysicalPlan implement(LogicalPlan plan) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java index d935ea7029e..97ce0592c48 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java @@ -9,6 +9,7 @@ import static org.opensearch.sql.common.setting.Settings.Key.CALCITE_PUSHDOWN_ROWCOUNT_ESTIMATION_FACTOR; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.stream.Stream; import lombok.Getter; @@ -45,6 +46,16 @@ import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.storage.OpenSearchIndex; +import org.opensearch.sql.opensearch.storage.scan.context.AbstractAction; +import org.opensearch.sql.opensearch.storage.scan.context.AggPushDownAction; +import org.opensearch.sql.opensearch.storage.scan.context.AggregationBuilderAction; +import org.opensearch.sql.opensearch.storage.scan.context.FilterDigest; +import org.opensearch.sql.opensearch.storage.scan.context.LimitDigest; +import org.opensearch.sql.opensearch.storage.scan.context.OSRequestBuilderAction; +import org.opensearch.sql.opensearch.storage.scan.context.PushDownContext; +import org.opensearch.sql.opensearch.storage.scan.context.PushDownOperation; +import org.opensearch.sql.opensearch.storage.scan.context.PushDownType; +import org.opensearch.sql.opensearch.storage.scan.context.RareTopDigest; /** An abstract relational operator representing a scan of an OpenSearchIndex type. */ @Getter @@ -105,12 +116,23 @@ public double estimateRowCount(RelMetadataQuery mq) { switch (operation.type()) { case AGGREGATION -> mq.getRowCount((RelNode) operation.digest()); case PROJECT, SORT -> rowCount; + case SORT_AGG_METRICS -> NumberUtil.min( + rowCount, osIndex.getBucketSize().doubleValue()); // Refer the org.apache.calcite.rel.metadata.RelMdRowCount case COLLAPSE -> rowCount / 10; case FILTER, SCRIPT -> NumberUtil.multiply( rowCount, RelMdUtil.guessSelectivity(((FilterDigest) operation.digest()).condition())); case LIMIT -> Math.min(rowCount, ((LimitDigest) operation.digest()).limit()); + case RARE_TOP -> { + /** similar to {@link Aggregate#estimateRowCount(RelMetadataQuery)} */ + final RareTopDigest digest = (RareTopDigest) operation.digest(); + int factor = digest.number(); + final int groupCount = digest.byList().size(); + yield groupCount == 0 + ? factor + : factor * rowCount * (1.0 - Math.pow(.5, groupCount)); + } }, (a, b) -> null); } @@ -141,6 +163,10 @@ public double estimateRowCount(RelMetadataQuery mq) { // Ignored Project in cost accumulation, but it will affect the external cost case PROJECT -> {} case SORT -> dCpu += dRows; + case SORT_AGG_METRICS -> { + dRows = dRows * .9 / 10; // *.9 because always bucket IS_NOT_NULL + dCpu += dRows; + } // Refer the org.apache.calcite.rel.metadata.RelMdRowCount.getRowCount(Aggregate rel,...) case COLLAPSE -> { dRows = dRows / 10; @@ -162,8 +188,15 @@ public double estimateRowCount(RelMetadataQuery mq) { // Because we'd like to push down LIMIT even when the fetch in LIMIT is greater than // dRows. case LIMIT -> dRows = Math.min(dRows, ((LimitDigest) operation.digest()).limit()) - 1; + case RARE_TOP -> { + /** similar to {@link Aggregate#computeSelfCost(RelOptPlanner, RelMetadataQuery)} */ + final RareTopDigest digest = (RareTopDigest) operation.digest(); + int factor = digest.number(); + final int groupCount = digest.byList().size(); + dRows = groupCount == 0 ? factor : factor * dRows * (1.0 - Math.pow(.5, groupCount)); + dCpu += dRows * 1.125f; + } } - ; } // Add the external cost to introduce the effect from FILTER, LIMIT and PROJECT. dCpu += dRows * getRowType().getFieldList().size(); @@ -208,31 +241,60 @@ protected abstract AbstractCalciteIndexScan buildScan( RelDataType schema, PushDownContext pushDownContext); - private List getCollationNames(List collations) { + protected List getCollationNames(List collations) { return collations.stream() .map(collation -> getRowType().getFieldNames().get(collation.getFieldIndex())) .toList(); } /** - * Check if the sort by collations contains any aggregators that are pushed down. E.g. In `stats - * avg(age) as avg_age by state | sort avg_age`, the sort clause has `avg_age` which is an - * aggregator. The function will return true in this case. + * Check if all sort-by collations equal aggregators that are pushed down. E.g. In `stats avg(age) + * as avg_age, sum(age) as sum_age by state | sort avg_age, sum_age`, the sort keys `avg_age`, + * `sum_age` which equal the pushed down aggregators `avg(age)`, `sum(age)`. + * + * @param collations List of collation names to check against aggregators. + * @return True if all collation names match all aggregator output, false otherwise. + */ + protected boolean isAllCollationNamesEqualAggregators(List collations) { + Stream aggregates = + pushDownContext.stream() + .filter(action -> action.type() == PushDownType.AGGREGATION) + .map(action -> ((LogicalAggregate) action.digest())); + return aggregates + .map(aggregate -> isAllCollationNamesEqualAggregators(aggregate, collations)) + .reduce(false, Boolean::logicalOr); + } + + private boolean isAllCollationNamesEqualAggregators( + LogicalAggregate aggregate, List collations) { + List fieldNames = aggregate.getRowType().getFieldNames(); + // The output fields of the aggregate are in the format of + // [...grouping fields, ...aggregator fields], so we set an offset to skip + // the grouping fields. + int groupOffset = aggregate.getGroupSet().cardinality(); + List fieldsWithoutGrouping = fieldNames.subList(groupOffset, fieldNames.size()); + return new HashSet<>(collations).equals(new HashSet<>(fieldsWithoutGrouping)); + } + + /** + * Check if any sort-by collations is in aggregators that are pushed down. E.g. In `stats avg(age) + * as avg_age by state | sort avg_age`, the sort clause has `avg_age` which is an aggregator. The + * function will return true in this case. * * @param collations List of collation names to check against aggregators. * @return True if any collation name matches an aggregator output, false otherwise. */ - private boolean hasAggregatorInSortBy(List collations) { + protected boolean isAnyCollationNameInAggregators(List collations) { Stream aggregates = pushDownContext.stream() .filter(action -> action.type() == PushDownType.AGGREGATION) .map(action -> ((LogicalAggregate) action.digest())); return aggregates - .map(aggregate -> isAnyCollationNameInAggregateOutput(aggregate, collations)) + .map(aggregate -> isAnyCollationNameInAggregators(aggregate, collations)) .reduce(false, Boolean::logicalOr); } - private static boolean isAnyCollationNameInAggregateOutput( + private boolean isAnyCollationNameInAggregators( LogicalAggregate aggregate, List collations) { List fieldNames = aggregate.getRowType().getFieldNames(); // The output fields of the aggregate are in the format of @@ -253,25 +315,14 @@ private static boolean isAnyCollationNameInAggregateOutput( public AbstractCalciteIndexScan pushDownSort(List collations) { try { List collationNames = getCollationNames(collations); - if (getPushDownContext().isAggregatePushed() && hasAggregatorInSortBy(collationNames)) { + if (getPushDownContext().isAggregatePushed() + && isAnyCollationNameInAggregators(collationNames)) { // If aggregation is pushed down, we cannot push down sorts where its by fields contain // aggregators. return null; } - - // Propagate the sort to the new scan RelTraitSet traitsWithCollations = getTraitSet().plus(RelCollations.of(collations)); - AbstractCalciteIndexScan newScan = - buildScan( - getCluster(), - traitsWithCollations, - hints, - table, - osIndex, - getRowType(), - // Existing collations are overridden (discarded) by the new collations, - pushDownContext.cloneWithoutSort()); - + PushDownContext pushDownContextWithoutSort = this.pushDownContext.cloneWithoutSort(); AbstractAction action; Object digest; if (pushDownContext.isAggregatePushed()) { @@ -281,7 +332,27 @@ public AbstractCalciteIndexScan pushDownSort(List collations) aggAction -> aggAction.pushDownSortIntoAggBucket(collations, getRowType().getFieldNames()); digest = collations; + pushDownContextWithoutSort.add(PushDownType.SORT, digest, action); + return buildScan( + getCluster(), + traitsWithCollations, + hints, + table, + osIndex, + getRowType(), + pushDownContextWithoutSort.clone()); } else { + // Propagate the sort to the new scan + AbstractCalciteIndexScan newScan = + buildScan( + getCluster(), + traitsWithCollations, + hints, + table, + osIndex, + getRowType(), + // Existing collations are overridden (discarded) by the new collations, + pushDownContextWithoutSort); List> builders = new ArrayList<>(); for (RelFieldCollation collation : collations) { int index = collation.getFieldIndex(); @@ -310,9 +381,9 @@ public AbstractCalciteIndexScan pushDownSort(List collations) } action = (OSRequestBuilderAction) requestBuilder -> requestBuilder.pushDownSort(builders); digest = builders.toString(); + newScan.pushDownContext.add(PushDownType.SORT, digest, action); + return newScan; } - newScan.pushDownContext.add(PushDownType.SORT, digest, action); - return newScan; } catch (Exception e) { if (LOG.isDebugEnabled()) { LOG.debug("Cannot pushdown the sort {}", getCollationNames(collations), e); @@ -320,4 +391,26 @@ public AbstractCalciteIndexScan pushDownSort(List collations) } return null; } + + /** + * CalciteOpenSearchIndexScan doesn't allow push-down anymore (except Sort under some strict + * condition) after Aggregate push-down. + */ + public boolean noAggregatePushed() { + if (this.getPushDownContext().isAggregatePushed()) return false; + final RelOptTable table = this.getTable(); + return table.unwrap(OpenSearchIndex.class) != null; + } + + public boolean isLimitPushed() { + return this.getPushDownContext().isLimitPushed(); + } + + public boolean isMetricsOrderPushed() { + return this.getPushDownContext().isMeasureOrderPushed(); + } + + public boolean isTopKPushed() { + return this.getPushDownContext().isTopKPushed(); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java index 7fb32cc761a..20d8a6c34fd 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java @@ -31,6 +31,8 @@ import org.opensearch.sql.calcite.plan.Scannable; import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder; import org.opensearch.sql.opensearch.storage.OpenSearchIndex; +import org.opensearch.sql.opensearch.storage.scan.context.PushDownContext; +import org.opensearch.sql.opensearch.util.OpenSearchRelOptUtil; /** The physical relational operator representing a scan of an OpenSearchIndex type. */ public class CalciteEnumerableIndexScan extends AbstractCalciteIndexScan @@ -87,9 +89,14 @@ public Result implement(EnumerableRelImplementor implementor, Prefer pref) { * let's follow this convention to apply the optimization here and ensure `scan` method * returns the correct data format for single column rows. * See {@link OpenSearchIndexEnumerator} + * Besides, we replace all dots in fields to avoid the Calcite codegen bug. + * https://github.com/opensearch-project/sql/issues/4619 */ PhysType physType = - PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray()); + PhysTypeImpl.of( + implementor.getTypeFactory(), + OpenSearchRelOptUtil.replaceDot(getCluster().getTypeFactory(), getRowType()), + pref.preferArray()); Expression scanOperator = implementor.stash(this, CalciteEnumerableIndexScan.class); return implementor.result(physType, Blocks.toBlock(Expressions.call(scanOperator, "scan"))); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java index ee9f6be144c..a6ff44719d9 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java @@ -24,8 +24,8 @@ import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.hint.RelHint; -import org.apache.calcite.rel.logical.LogicalFilter; import org.apache.calcite.rel.logical.LogicalSort; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; @@ -33,26 +33,34 @@ import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.sql.type.SqlTypeName; import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.search.aggregations.AggregationBuilder; -import org.opensearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; -import org.opensearch.search.aggregations.metrics.ValueCountAggregationBuilder; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; +import org.opensearch.sql.ast.expression.Argument; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; -import org.opensearch.sql.opensearch.planner.physical.EnumerableIndexScanRule; -import org.opensearch.sql.opensearch.planner.physical.OpenSearchIndexRules; +import org.opensearch.sql.opensearch.planner.rules.EnumerableIndexScanRule; +import org.opensearch.sql.opensearch.planner.rules.OpenSearchIndexRules; import org.opensearch.sql.opensearch.request.AggregateAnalyzer; import org.opensearch.sql.opensearch.request.PredicateAnalyzer; import org.opensearch.sql.opensearch.request.PredicateAnalyzer.QueryExpression; import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; import org.opensearch.sql.opensearch.storage.OpenSearchIndex; +import org.opensearch.sql.opensearch.storage.scan.context.AbstractAction; +import org.opensearch.sql.opensearch.storage.scan.context.AggPushDownAction; +import org.opensearch.sql.opensearch.storage.scan.context.AggregationBuilderAction; +import org.opensearch.sql.opensearch.storage.scan.context.FilterDigest; +import org.opensearch.sql.opensearch.storage.scan.context.LimitDigest; +import org.opensearch.sql.opensearch.storage.scan.context.OSRequestBuilderAction; +import org.opensearch.sql.opensearch.storage.scan.context.PushDownContext; +import org.opensearch.sql.opensearch.storage.scan.context.PushDownType; +import org.opensearch.sql.opensearch.storage.scan.context.RareTopDigest; /** The logical relational operator representing a scan of an OpenSearchIndex type. */ @Getter @@ -95,6 +103,11 @@ protected AbstractCalciteIndexScan buildScan( cluster, traitSet, hints, table, osIndex, schema, pushDownContext); } + public CalciteLogicalIndexScan copy() { + return new CalciteLogicalIndexScan( + getCluster(), traitSet, hints, table, osIndex, schema, pushDownContext.clone()); + } + public CalciteLogicalIndexScan copyWithNewSchema(RelDataType schema) { // Do shallow copy for requestBuilder, thus requestBuilder among different plans produced in the // optimization process won't affect each other. @@ -102,6 +115,11 @@ public CalciteLogicalIndexScan copyWithNewSchema(RelDataType schema) { getCluster(), traitSet, hints, table, osIndex, schema, pushDownContext.clone()); } + public CalciteLogicalIndexScan copyWithNewTraitSet(RelTraitSet traitSet) { + return new CalciteLogicalIndexScan( + getCluster(), traitSet, hints, table, osIndex, schema, pushDownContext.clone()); + } + @Override public void register(RelOptPlanner planner) { super.register(planner); @@ -119,17 +137,17 @@ public void register(RelOptPlanner planner) { public AbstractRelNode pushDownFilter(Filter filter) { try { - RelDataType rowType = filter.getRowType(); - CalciteLogicalIndexScan newScan = this.copyWithNewSchema(filter.getRowType()); + RelDataType rowType = this.getRowType(); List schema = this.getRowType().getFieldNames(); Map fieldTypes = - this.osIndex.getFieldTypes().entrySet().stream() + this.osIndex.getAllFieldTypes().entrySet().stream() .filter(entry -> schema.contains(entry.getKey())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); QueryExpression queryExpression = PredicateAnalyzer.analyzeExpression( filter.getCondition(), schema, fieldTypes, rowType, getCluster()); // TODO: handle the case where condition contains a score function + CalciteLogicalIndexScan newScan = this.copy(); newScan.pushDownContext.add( queryExpression.getScriptCount() > 0 ? PushDownType.SCRIPT : PushDownType.FILTER, new FilterDigest( @@ -272,6 +290,54 @@ private RelTraitSet reIndexCollations(List selectedColumns) { return newTraitSet; } + public CalciteLogicalIndexScan pushDownSortAggregateMeasure(Sort sort) { + try { + if (!pushDownContext.isAggregatePushed()) return null; + List aggregationBuilders = + pushDownContext.getAggPushDownAction().getAggregationBuilder().getLeft(); + if (aggregationBuilders.size() != 1) { + return null; + } + if (!(aggregationBuilders.getFirst() instanceof CompositeAggregationBuilder)) { + return null; + } +// FIXME: Needs Optimised Index setting check +// List collationNames = getCollationNames(sort.getCollation().getFieldCollations()); +// if (!isAllCollationNamesEqualAggregators(collationNames)) { +// return null; +// } + CalciteLogicalIndexScan newScan = copyWithNewTraitSet(sort.getTraitSet()); + AbstractAction newAction = + (AggregationBuilderAction) + aggAction -> + aggAction.rePushDownSortAggMeasure( + sort.getCollation().getFieldCollations(), rowType.getFieldNames()); + Object digest = sort.getCollation().getFieldCollations(); + newScan.pushDownContext.add(PushDownType.SORT_AGG_METRICS, digest, newAction); + return newScan; + } catch (Exception e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Cannot pushdown the sort aggregate {}", sort, e); + } + } + return null; + } + + public CalciteLogicalIndexScan pushDownRareTop(Project project, RareTopDigest digest) { + try { + CalciteLogicalIndexScan newScan = copyWithNewSchema(project.getRowType()); + AbstractAction newAction = + (AggregationBuilderAction) aggAction -> aggAction.rePushDownRareTop(digest); + newScan.pushDownContext.add(PushDownType.RARE_TOP, digest, newAction); + return newScan; + } catch (Exception e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Cannot pushdown {}", digest, e); + } + return null; + } + } + public AbstractRelNode pushDownAggregate(Aggregate aggregate, Project project) { try { CalciteLogicalIndexScan newScan = @@ -284,11 +350,25 @@ public AbstractRelNode pushDownAggregate(Aggregate aggregate, Project project) { aggregate.getRowType(), // Aggregation will eliminate all collations. pushDownContext.cloneWithoutSort()); - Map fieldTypes = this.osIndex.getFieldTypes(); + List schema = this.getRowType().getFieldNames(); + Map fieldTypes = + this.osIndex.getAllFieldTypes().entrySet().stream() + .filter(entry -> schema.contains(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); List outputFields = aggregate.getRowType().getFieldNames(); + int bucketSize = osIndex.getBucketSize(); + boolean bucketNullable = + Boolean.parseBoolean( + aggregate.getHints().stream() + .filter(hits -> hits.hintName.equals("stats_args")) + .map(hint -> hint.kvOptions.getOrDefault(Argument.BUCKET_NULLABLE, "true")) + .findFirst() + .orElseGet(() -> "true")); + AggregateAnalyzer.AggregateBuilderHelper helper = + new AggregateAnalyzer.AggregateBuilderHelper( + getRowType(), fieldTypes, getCluster(), bucketNullable, bucketSize); final Pair, OpenSearchAggregationResponseParser> aggregationBuilder = - AggregateAnalyzer.analyze( - aggregate, project, getRowType(), fieldTypes, outputFields, getCluster()); + AggregateAnalyzer.analyze(aggregate, project, outputFields, helper); Map extendedTypeMapping = aggregate.getRowType().getFieldList().stream() .collect( @@ -304,29 +384,9 @@ public AbstractRelNode pushDownAggregate(Aggregate aggregate, Project project) { extendedTypeMapping, outputFields.subList(0, aggregate.getGroupSet().cardinality())); newScan.pushDownContext.add(PushDownType.AGGREGATION, aggregate, action); - if (aggregationBuilder.getLeft().size() == 1 - && aggregationBuilder.getLeft().getFirst() - instanceof AutoDateHistogramAggregationBuilder autoDateHistogram) { - // If it's auto_date_histogram, filter the empty bucket by using the first aggregate metrics - RexBuilder rexBuilder = getCluster().getRexBuilder(); - AggregationBuilder aggregationBuilders = - autoDateHistogram.getSubAggregations().stream().toList().getFirst(); - RexNode condition = - aggregationBuilders instanceof ValueCountAggregationBuilder - ? rexBuilder.makeCall( - SqlStdOperatorTable.GREATER_THAN, - rexBuilder.makeInputRef(newScan, 1), - rexBuilder.makeLiteral( - 0, rexBuilder.getTypeFactory().createSqlType(SqlTypeName.INTEGER))) - : rexBuilder.makeCall( - SqlStdOperatorTable.IS_NOT_NULL, rexBuilder.makeInputRef(newScan, 1)); - return LogicalFilter.create(newScan, condition); - } return newScan; } catch (Exception e) { - if (LOG.isDebugEnabled()) { - LOG.debug("Cannot pushdown the aggregate {}", aggregate, e); - } + LOG.info("Cannot pushdown the aggregate {}", aggregate, e); } return null; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/PushDownContext.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/PushDownContext.java deleted file mode 100644 index 9819f893e1e..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/PushDownContext.java +++ /dev/null @@ -1,342 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.scan; - -import com.google.common.collect.Iterators; -import java.util.AbstractCollection; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.stream.IntStream; -import lombok.Getter; -import org.apache.calcite.rel.RelFieldCollation; -import org.apache.calcite.rel.RelFieldCollation.Direction; -import org.apache.calcite.rel.RelFieldCollation.NullDirection; -import org.apache.calcite.rex.RexNode; -import org.apache.commons.lang3.tuple.Pair; -import org.jetbrains.annotations.NotNull; -import org.opensearch.search.aggregations.AggregationBuilder; -import org.opensearch.search.aggregations.AggregationBuilders; -import org.opensearch.search.aggregations.AggregatorFactories.Builder; -import org.opensearch.search.aggregations.BucketOrder; -import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; -import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; -import org.opensearch.search.aggregations.bucket.missing.MissingOrder; -import org.opensearch.search.aggregations.bucket.terms.MultiTermsAggregationBuilder; -import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; -import org.opensearch.search.sort.SortOrder; -import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; -import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder; -import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; -import org.opensearch.sql.opensearch.storage.OpenSearchIndex; - -@Getter -public class PushDownContext extends AbstractCollection { - private final OpenSearchIndex osIndex; - private final OpenSearchRequestBuilder requestBuilder; - private ArrayDeque operationsForRequestBuilder; - - private boolean isAggregatePushed = false; - private AggPushDownAction aggPushDownAction; - private ArrayDeque operationsForAgg; - - private boolean isLimitPushed = false; - private boolean isProjectPushed = false; - - public PushDownContext(OpenSearchIndex osIndex) { - this.osIndex = osIndex; - this.requestBuilder = osIndex.createRequestBuilder(); - } - - @Override - public PushDownContext clone() { - PushDownContext newContext = new PushDownContext(osIndex); - newContext.addAll(this); - return newContext; - } - - /** - * Create a new {@link PushDownContext} without the collation action. - * - * @return A new push-down context without the collation action. - */ - public PushDownContext cloneWithoutSort() { - PushDownContext newContext = new PushDownContext(osIndex); - for (PushDownOperation action : this) { - if (action.type() != PushDownType.SORT) { - newContext.add(action); - } - } - return newContext; - } - - @NotNull - @Override - public Iterator iterator() { - if (operationsForRequestBuilder == null) { - return Collections.emptyIterator(); - } else if (operationsForAgg == null) { - return operationsForRequestBuilder.iterator(); - } else { - return Iterators.concat(operationsForRequestBuilder.iterator(), operationsForAgg.iterator()); - } - } - - @Override - public int size() { - return (operationsForRequestBuilder == null ? 0 : operationsForRequestBuilder.size()) - + (operationsForAgg == null ? 0 : operationsForAgg.size()); - } - - ArrayDeque getOperationsForRequestBuilder() { - if (operationsForRequestBuilder == null) { - this.operationsForRequestBuilder = new ArrayDeque<>(); - } - return operationsForRequestBuilder; - } - - ArrayDeque getOperationsForAgg() { - if (operationsForAgg == null) { - this.operationsForAgg = new ArrayDeque<>(); - } - return operationsForAgg; - } - - @Override - public boolean add(PushDownOperation operation) { - if (operation.type() == PushDownType.AGGREGATION) { - isAggregatePushed = true; - this.aggPushDownAction = (AggPushDownAction) operation.action(); - } - if (operation.type() == PushDownType.LIMIT) { - isLimitPushed = true; - } - if (operation.type() == PushDownType.PROJECT) { - isProjectPushed = true; - } - operation.action().transform(this, operation); - return true; - } - - void add(PushDownType type, Object digest, AbstractAction action) { - add(new PushDownOperation(type, digest, action)); - } - - public boolean containsDigest(Object digest) { - return this.stream().anyMatch(action -> action.digest().equals(digest)); - } - - public OpenSearchRequestBuilder createRequestBuilder() { - OpenSearchRequestBuilder newRequestBuilder = osIndex.createRequestBuilder(); - if (operationsForRequestBuilder != null) { - operationsForRequestBuilder.forEach( - operation -> ((OSRequestBuilderAction) operation.action()).apply(newRequestBuilder)); - } - return newRequestBuilder; - } -} - -enum PushDownType { - FILTER, - PROJECT, - AGGREGATION, - SORT, - LIMIT, - SCRIPT, - COLLAPSE - // HIGHLIGHT, - // NESTED -} - -/** - * Represents a push down operation that can be applied to an OpenSearchRequestBuilder. - * - * @param type PushDownType enum - * @param digest the digest of the pushed down operator - * @param action the lambda action to apply on the OpenSearchRequestBuilder - */ -record PushDownOperation(PushDownType type, Object digest, AbstractAction action) { - public String toString() { - return type + "->" + digest; - } -} - -interface AbstractAction { - void apply(T target); - - void transform(PushDownContext context, PushDownOperation operation); -} - -interface OSRequestBuilderAction extends AbstractAction { - default void transform(PushDownContext context, PushDownOperation operation) { - apply(context.getRequestBuilder()); - context.getOperationsForRequestBuilder().add(operation); - } -} - -interface AggregationBuilderAction extends AbstractAction { - default void transform(PushDownContext context, PushDownOperation operation) { - apply(context.getAggPushDownAction()); - context.getOperationsForAgg().add(operation); - } -} - -record FilterDigest(int scriptCount, RexNode condition) { - @Override - public String toString() { - return condition.toString(); - } -} - -record LimitDigest(int limit, int offset) { - @Override - public String toString() { - return offset == 0 ? String.valueOf(limit) : "[" + limit + " from " + offset + "]"; - } -} - -// TODO: shall we do deep copy for this action since it's mutable? -class AggPushDownAction implements OSRequestBuilderAction { - - private Pair, OpenSearchAggregationResponseParser> aggregationBuilder; - private final Map extendedTypeMapping; - @Getter private final long scriptCount; - // Record the output field names of all buckets as the sequence of buckets - private List bucketNames; - - public AggPushDownAction( - Pair, OpenSearchAggregationResponseParser> aggregationBuilder, - Map extendedTypeMapping, - List bucketNames) { - this.aggregationBuilder = aggregationBuilder; - this.extendedTypeMapping = extendedTypeMapping; - this.scriptCount = - aggregationBuilder.getLeft().stream().filter(this::isScriptAggBuilder).count(); - this.bucketNames = bucketNames; - } - - private boolean isScriptAggBuilder(AggregationBuilder aggBuilder) { - return aggBuilder instanceof ValuesSourceAggregationBuilder valueSourceAgg - && valueSourceAgg.script() != null; - } - - @Override - public void apply(OpenSearchRequestBuilder requestBuilder) { - requestBuilder.pushDownAggregation(aggregationBuilder); - requestBuilder.pushTypeMapping(extendedTypeMapping); - } - - public void pushDownSortIntoAggBucket( - List collations, List fieldNames) { - // aggregationBuilder.getLeft() could be empty when count agg optimization works - if (aggregationBuilder.getLeft().isEmpty()) return; - AggregationBuilder builder = aggregationBuilder.getLeft().getFirst(); - List selected = new ArrayList<>(collations.size()); - if (builder instanceof CompositeAggregationBuilder compositeAggBuilder) { - // It will always use a single CompositeAggregationBuilder for the aggregation with GroupBy - // See {@link AggregateAnalyzer} - List> buckets = compositeAggBuilder.sources(); - List> newBuckets = new ArrayList<>(buckets.size()); - List newBucketNames = new ArrayList<>(buckets.size()); - // Have to put the collation required buckets first, then the rest of buckets. - collations.forEach( - collation -> { - /* - Must find the bucket by field name because: - 1. The sequence of buckets may have changed after sort push-down. - 2. The schema of scan operator may be inconsistent with the sequence of buckets - after project push-down. - */ - String bucketName = fieldNames.get(collation.getFieldIndex()); - CompositeValuesSourceBuilder bucket = buckets.get(bucketNames.indexOf(bucketName)); - Direction direction = collation.getDirection(); - NullDirection nullDirection = collation.nullDirection; - SortOrder order = - Direction.DESCENDING.equals(direction) ? SortOrder.DESC : SortOrder.ASC; - if (bucket.missingBucket()) { - MissingOrder missingOrder = - switch (nullDirection) { - case FIRST -> MissingOrder.FIRST; - case LAST -> MissingOrder.LAST; - default -> MissingOrder.DEFAULT; - }; - bucket.missingOrder(missingOrder); - } - newBuckets.add(bucket.order(order)); - newBucketNames.add(bucketName); - selected.add(bucketName); - }); - IntStream.range(0, buckets.size()) - .mapToObj(fieldNames::get) - .filter(name -> !selected.contains(name)) - .forEach( - name -> { - newBuckets.add(buckets.get(bucketNames.indexOf(name))); - newBucketNames.add(name); - }); - Builder newAggBuilder = new Builder(); - compositeAggBuilder.getSubAggregations().forEach(newAggBuilder::addAggregator); - aggregationBuilder = - Pair.of( - Collections.singletonList( - AggregationBuilders.composite("composite_buckets", newBuckets) - .subAggregations(newAggBuilder) - .size(compositeAggBuilder.size())), - aggregationBuilder.getRight()); - bucketNames = newBucketNames; - } - if (builder instanceof TermsAggregationBuilder termsAggBuilder) { - termsAggBuilder.order(BucketOrder.key(!collations.getFirst().getDirection().isDescending())); - } - // TODO for MultiTermsAggregationBuilder - } - - /** - * Check if the limit can be pushed down into aggregation bucket when the limit size is less than - * bucket number. - */ - public boolean pushDownLimitIntoBucketSize(Integer size) { - // aggregationBuilder.getLeft() could be empty when count agg optimization works - if (aggregationBuilder.getLeft().isEmpty()) return false; - AggregationBuilder builder = aggregationBuilder.getLeft().getFirst(); - if (builder instanceof CompositeAggregationBuilder compositeAggBuilder) { - if (size < compositeAggBuilder.size()) { - compositeAggBuilder.size(size); - return true; - } else { - return false; - } - } - if (builder instanceof TermsAggregationBuilder termsAggBuilder) { - if (size < termsAggBuilder.size()) { - termsAggBuilder.size(size); - return true; - } else { - return false; - } - } - if (builder instanceof MultiTermsAggregationBuilder multiTermsAggBuilder) { - if (size < multiTermsAggBuilder.size()) { - multiTermsAggBuilder.size(size); - return true; - } else { - return false; - } - } - // now we only have Composite, Terms and MultiTerms bucket aggregations, - // add code here when we could support more in the future. - if (builder instanceof ValuesSourceAggregationBuilder.LeafOnly) { - // Note: all metric aggregations will be treated as pushed since it generates only one row. - return true; - } - throw new OpenSearchRequestBuilder.PushDownUnSupportedException( - "Unknown aggregation builder " + builder.getClass().getSimpleName()); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AbstractAction.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AbstractAction.java new file mode 100644 index 00000000000..65ef6233ffb --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AbstractAction.java @@ -0,0 +1,23 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.scan.context; + +/** + * A lambda action to apply on the target T + * + * @param the target type + */ +public interface AbstractAction { + void apply(T target); + + /** + * Apply the action on the target T and add the operation to the context + * + * @param context the context to add the operation to + * @param operation the operation to add to the context + */ + void transform(PushDownContext context, PushDownOperation operation); +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AggPushDownAction.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AggPushDownAction.java new file mode 100644 index 00000000000..7a32458cb74 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AggPushDownAction.java @@ -0,0 +1,448 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.scan.context; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.apache.calcite.rel.RelFieldCollation; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.search.aggregations.AbstractAggregationBuilder; +import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.AggregatorFactories; +import org.opensearch.search.aggregations.BucketOrder; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; +import org.opensearch.search.aggregations.bucket.composite.DateHistogramValuesSourceBuilder; +import org.opensearch.search.aggregations.bucket.composite.HistogramValuesSourceBuilder; +import org.opensearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder; +import org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; +import org.opensearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder; +import org.opensearch.search.aggregations.bucket.missing.MissingOrder; +import org.opensearch.search.aggregations.bucket.terms.MultiTermsAggregationBuilder; +import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.opensearch.search.aggregations.support.MultiTermsValuesSourceConfig; +import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder; +import org.opensearch.sql.opensearch.response.agg.BucketAggregationParser; +import org.opensearch.sql.opensearch.response.agg.CompositeAggregationParser; +import org.opensearch.sql.opensearch.response.agg.MetricParserHelper; +import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; + +/** A lambda aggregation pushdown action to apply on the {@link OpenSearchRequestBuilder} */ +@Getter +@EqualsAndHashCode +public class AggPushDownAction implements OSRequestBuilderAction { + + private Pair, OpenSearchAggregationResponseParser> aggregationBuilder; + private final Map extendedTypeMapping; + private final long scriptCount; + // Record the output field names of all buckets as the sequence of buckets + private List bucketNames; + + public AggPushDownAction( + Pair, OpenSearchAggregationResponseParser> aggregationBuilder, + Map extendedTypeMapping, + List bucketNames) { + this.aggregationBuilder = aggregationBuilder; + this.extendedTypeMapping = extendedTypeMapping; + this.scriptCount = + aggregationBuilder.getLeft().stream().mapToInt(AggPushDownAction::getScriptCount).sum(); + this.bucketNames = bucketNames; + } + + private static int getScriptCount(AggregationBuilder aggBuilder) { + if (aggBuilder instanceof ValuesSourceAggregationBuilder + && ((ValuesSourceAggregationBuilder) aggBuilder).script() != null) return 1; + if (aggBuilder instanceof CompositeAggregationBuilder) { + CompositeAggregationBuilder compositeAggBuilder = (CompositeAggregationBuilder) aggBuilder; + int sourceScriptCount = + compositeAggBuilder.sources().stream() + .mapToInt(source -> source.script() != null ? 1 : 0) + .sum(); + int subAggScriptCount = + compositeAggBuilder.getSubAggregations().stream() + .mapToInt(AggPushDownAction::getScriptCount) + .sum(); + return sourceScriptCount + subAggScriptCount; + } + return 0; + } + + @Override + public void apply(OpenSearchRequestBuilder requestBuilder) { + requestBuilder.pushDownAggregation(aggregationBuilder); + requestBuilder.pushTypeMapping(extendedTypeMapping); + } + + /** Convert a {@link CompositeAggregationParser} to {@link BucketAggregationParser} */ + private BucketAggregationParser convertTo(OpenSearchAggregationResponseParser parser) { + if (parser instanceof BucketAggregationParser) { + return (BucketAggregationParser) parser; + } else if (parser instanceof CompositeAggregationParser) { + MetricParserHelper helper = ((CompositeAggregationParser) parser).getMetricsParser(); + return new BucketAggregationParser( + helper.getMetricParserMap().values().stream().toList(), helper.getCountAggNameList()); + } else { + throw new IllegalStateException("Unexpected parser type: " + parser.getClass()); + } + } + + private String multiTermsBucketNameAsString(CompositeAggregationBuilder composite) { + return composite.sources().stream() + .map(TermsValuesSourceBuilder.class::cast) + .map(TermsValuesSourceBuilder::name) + .collect(Collectors.joining("|")); // PIPE cannot be used in identifier + } + + /** Re-pushdown a sort aggregation measure to replace the pushed composite aggregation */ + public void rePushDownSortAggMeasure( + List collations, List fieldNames) { + if (aggregationBuilder.getLeft().isEmpty()) return; + AggregationBuilder builder = aggregationBuilder.getLeft().getFirst(); + if (builder instanceof CompositeAggregationBuilder composite) { + String path = getAggregationPath(collations, fieldNames, composite); + BucketOrder bucketOrder = + collations.get(0).getDirection() == RelFieldCollation.Direction.ASCENDING + ? BucketOrder.aggregation(path, true) + : BucketOrder.aggregation(path, false); + + if (composite.sources().size() == 1) { + if (composite.sources().get(0) instanceof TermsValuesSourceBuilder terms + /*&& !terms.missingBucket()*/) { // FIXME: Needs Optimised Index setting check + TermsAggregationBuilder termsBuilder = + buildTermsAggregationBuilder(terms, bucketOrder, composite.size()); + attachSubAggregations(composite.getSubAggregations(), path, termsBuilder); + aggregationBuilder = + Pair.of( + Collections.singletonList(termsBuilder), + convertTo(aggregationBuilder.getRight())); + return; + } else if (composite.sources().get(0) + instanceof DateHistogramValuesSourceBuilder dateHisto) { + DateHistogramAggregationBuilder dateHistoBuilder = + buildDateHistogramAggregationBuilder(dateHisto, bucketOrder); + attachSubAggregations(composite.getSubAggregations(), path, dateHistoBuilder); + aggregationBuilder = + Pair.of( + Collections.singletonList(dateHistoBuilder), + convertTo(aggregationBuilder.getRight())); + return; + } else if (composite.sources().get(0) instanceof HistogramValuesSourceBuilder histo + && !histo.missingBucket()) { + HistogramAggregationBuilder histoBuilder = + buildHistogramAggregationBuilder(histo, bucketOrder); + attachSubAggregations(composite.getSubAggregations(), path, histoBuilder); + aggregationBuilder = + Pair.of( + Collections.singletonList(histoBuilder), + convertTo(aggregationBuilder.getRight())); + return; + } + } else { + if (composite.sources().stream() + .allMatch( + src -> src instanceof TermsValuesSourceBuilder terms /*&& !terms.missingBucket()*/)) { // FIXME: Needs Optimised Index setting check + // multi-term agg + MultiTermsAggregationBuilder multiTermsBuilder = + buildMultiTermsAggregationBuilder(composite); + attachSubAggregations(composite.getSubAggregations(), path, multiTermsBuilder); + aggregationBuilder = + Pair.of( + Collections.singletonList(multiTermsBuilder), + convertTo(aggregationBuilder.getRight())); + return; + } + } + throw new OpenSearchRequestBuilder.PushDownUnSupportedException( + "Cannot pushdown sort aggregate measure"); + } + } + + /** Re-pushdown a nested aggregation for rare/top to replace the pushed composite aggregation */ + public void rePushDownRareTop(RareTopDigest digest) { + if (aggregationBuilder.getLeft().isEmpty()) return; + AggregationBuilder builder = aggregationBuilder.getLeft().getFirst(); + if (builder instanceof CompositeAggregationBuilder composite) { + BucketOrder bucketOrder = + digest.direction() == RelFieldCollation.Direction.ASCENDING + ? BucketOrder.count(true) + : BucketOrder.count(false); + if (composite.sources().size() == 1) { + if (composite.sources().get(0) instanceof TermsValuesSourceBuilder terms + && !terms.missingBucket()) { + TermsAggregationBuilder termsBuilder = + buildTermsAggregationBuilder(terms, bucketOrder, digest.number()); + aggregationBuilder = + Pair.of( + Collections.singletonList(termsBuilder), + convertTo(aggregationBuilder.getRight())); + return; + } else if (composite.sources().get(0) + instanceof DateHistogramValuesSourceBuilder dateHisto) { + // for top/rare, only field can be used in by-clause, so this branch never accessed now + DateHistogramAggregationBuilder dateHistoBuilder = + buildDateHistogramAggregationBuilder(dateHisto, bucketOrder); + aggregationBuilder = + Pair.of( + Collections.singletonList(dateHistoBuilder), + convertTo(aggregationBuilder.getRight())); + return; + } else if (composite.sources().get(0) instanceof HistogramValuesSourceBuilder histo + && !histo.missingBucket()) { + // for top/rare, only field can be used in by-clause, so this branch never accessed now + HistogramAggregationBuilder histoBuilder = + buildHistogramAggregationBuilder(histo, bucketOrder); + aggregationBuilder = + Pair.of( + Collections.singletonList(histoBuilder), + convertTo(aggregationBuilder.getRight())); + return; + } + } else { + if (composite.sources().stream() + .allMatch( + src -> src instanceof TermsValuesSourceBuilder terms && !terms.missingBucket())) { + // nested term agg + TermsAggregationBuilder termsBuilder = null; + for (int i = 0; i < composite.sources().size(); i++) { + TermsValuesSourceBuilder terms = (TermsValuesSourceBuilder) composite.sources().get(i); + if (i == 0) { // first + termsBuilder = buildTermsAggregationBuilder(terms, null, 65535); + } else if (i == composite.sources().size() - 1) { // last + termsBuilder.subAggregation( + buildTermsAggregationBuilder(terms, bucketOrder, digest.number())); + } else { + termsBuilder.subAggregation(buildTermsAggregationBuilder(terms, null, 65535)); + } + } + aggregationBuilder = + Pair.of( + Collections.singletonList(termsBuilder), + convertTo(aggregationBuilder.getRight())); + return; + } + } + throw new OpenSearchRequestBuilder.PushDownUnSupportedException("Cannot pushdown " + digest); + } + } + + /** Build a {@link TermsAggregationBuilder} by {@link TermsValuesSourceBuilder} */ + private TermsAggregationBuilder buildTermsAggregationBuilder( + TermsValuesSourceBuilder terms, BucketOrder bucketOrder, int newSize) { + TermsAggregationBuilder termsBuilder = new TermsAggregationBuilder(terms.name()); + termsBuilder.size(newSize); + termsBuilder.field(terms.field()); + if (terms.userValuetypeHint() != null) { + termsBuilder.userValueTypeHint(terms.userValuetypeHint()); + } + if (bucketOrder != null) { + termsBuilder.order(bucketOrder); + } + return termsBuilder; + } + + /** Build a {@link DateHistogramAggregationBuilder} by {@link DateHistogramValuesSourceBuilder} */ + private DateHistogramAggregationBuilder buildDateHistogramAggregationBuilder( + DateHistogramValuesSourceBuilder dateHisto, BucketOrder bucketOrder) { + DateHistogramAggregationBuilder dateHistoBuilder = + new DateHistogramAggregationBuilder(dateHisto.name()); + dateHistoBuilder.field(dateHisto.field()); + try { + dateHistoBuilder.fixedInterval(dateHisto.getIntervalAsFixed()); + } catch (IllegalArgumentException e) { + dateHistoBuilder.calendarInterval(dateHisto.getIntervalAsCalendar()); + } + if (dateHisto.userValuetypeHint() != null) { + dateHistoBuilder.userValueTypeHint(dateHisto.userValuetypeHint()); + } + dateHistoBuilder.order(bucketOrder); + return dateHistoBuilder; + } + + /** Build a {@link HistogramAggregationBuilder} by {@link HistogramValuesSourceBuilder} */ + private HistogramAggregationBuilder buildHistogramAggregationBuilder( + HistogramValuesSourceBuilder histo, BucketOrder bucketOrder) { + HistogramAggregationBuilder histoBuilder = new HistogramAggregationBuilder(histo.name()); + histoBuilder.field(histo.field()); + histoBuilder.interval(histo.interval()); + if (histo.userValuetypeHint() != null) { + histoBuilder.userValueTypeHint(histo.userValuetypeHint()); + } + histoBuilder.order(bucketOrder); + return histoBuilder; + } + + /** Build a {@link MultiTermsAggregationBuilder} by {@link CompositeAggregationBuilder} */ + private MultiTermsAggregationBuilder buildMultiTermsAggregationBuilder( + CompositeAggregationBuilder composite) { + MultiTermsAggregationBuilder multiTermsBuilder = + new MultiTermsAggregationBuilder(multiTermsBucketNameAsString(composite)); + multiTermsBuilder.size(composite.size()); + multiTermsBuilder.terms( + composite.sources().stream() + .map(TermsValuesSourceBuilder.class::cast) + .map( + termValue -> { + MultiTermsValuesSourceConfig.Builder config = + new MultiTermsValuesSourceConfig.Builder(); + config.setFieldName(termValue.field()); + config.setUserValueTypeHint(termValue.userValuetypeHint()); + return config.build(); + }) + .toList()); + return multiTermsBuilder; + } + + private String getAggregationPath( + List collations, + List fieldNames, + CompositeAggregationBuilder composite) { + String path; + AggregationBuilder metric = composite.getSubAggregations().stream().findFirst().orElse(null); + if (metric == null) { + // count agg optimized, get the path name from field names + path = fieldNames.get(collations.get(0).getFieldIndex()); + } else if (metric instanceof ValuesSourceAggregationBuilder.LeafOnly) { + path = metric.getName(); + } else { + // we do not support pushdown sort aggregate measure for nested aggregation + throw new OpenSearchRequestBuilder.PushDownUnSupportedException( + "Cannot pushdown sort aggregate measure, composite.getSubAggregations() is not a" + + " LeafOnly"); + } + return path; + } + + private > T attachSubAggregations( + Collection subAggregations, String path, T aggregationBuilder) { + AggregatorFactories.Builder metricBuilder = new AggregatorFactories.Builder(); + if (subAggregations.isEmpty()) { + metricBuilder.addAggregator(AggregationBuilders.count(path).field("_index")); + } else { + subAggregations.forEach(metricBuilder::addAggregator); + } + aggregationBuilder.subAggregations(metricBuilder); + return aggregationBuilder; + } + + public void pushDownSortIntoAggBucket( + List collations, List fieldNames) { + // aggregationBuilder.getLeft() could be empty when count agg optimization works + if (aggregationBuilder.getLeft().isEmpty()) return; + AggregationBuilder builder = aggregationBuilder.getLeft().getFirst(); + List selected = new ArrayList<>(collations.size()); + if (builder instanceof CompositeAggregationBuilder compositeAggBuilder) { + // It will always use a single CompositeAggregationBuilder for the aggregation with GroupBy + // See {@link AggregateAnalyzer} + List> buckets = compositeAggBuilder.sources(); + List> newBuckets = new ArrayList<>(buckets.size()); + List newBucketNames = new ArrayList<>(buckets.size()); + // Have to put the collation required buckets first, then the rest of buckets. + collations.forEach( + collation -> { + /* + Must find the bucket by field name because: + 1. The sequence of buckets may have changed after sort push-down. + 2. The schema of scan operator may be inconsistent with the sequence of buckets + after project push-down. + */ + String bucketName = fieldNames.get(collation.getFieldIndex()); + CompositeValuesSourceBuilder bucket = buckets.get(bucketNames.indexOf(bucketName)); + RelFieldCollation.Direction direction = collation.getDirection(); + RelFieldCollation.NullDirection nullDirection = collation.nullDirection; + SortOrder order = + RelFieldCollation.Direction.DESCENDING.equals(direction) + ? SortOrder.DESC + : SortOrder.ASC; + if (bucket.missingBucket()) { + MissingOrder missingOrder = + switch (nullDirection) { + case FIRST -> MissingOrder.FIRST; + case LAST -> MissingOrder.LAST; + default -> MissingOrder.DEFAULT; + }; + bucket.missingOrder(missingOrder); + } + newBuckets.add(bucket.order(order)); + newBucketNames.add(bucketName); + selected.add(bucketName); + }); + buckets.stream() + .map(CompositeValuesSourceBuilder::name) + .filter(name -> !selected.contains(name)) + .forEach( + name -> { + newBuckets.add(buckets.get(bucketNames.indexOf(name))); + newBucketNames.add(name); + }); + AggregatorFactories.Builder newAggBuilder = new AggregatorFactories.Builder(); + compositeAggBuilder.getSubAggregations().forEach(newAggBuilder::addAggregator); + aggregationBuilder = + Pair.of( + Collections.singletonList( + AggregationBuilders.composite("composite_buckets", newBuckets) + .subAggregations(newAggBuilder) + .size(compositeAggBuilder.size())), + aggregationBuilder.getRight()); + bucketNames = newBucketNames; + } + if (builder instanceof TermsAggregationBuilder termsAggBuilder) { + termsAggBuilder.order(BucketOrder.key(!collations.getFirst().getDirection().isDescending())); + } + // TODO for MultiTermsAggregationBuilder + } + + /** + * Check if the limit can be pushed down into aggregation bucket when the limit size is less than + * bucket number. + */ + public boolean pushDownLimitIntoBucketSize(Integer size) { + // aggregationBuilder.getLeft() could be empty when count agg optimization works + if (aggregationBuilder.getLeft().isEmpty()) return false; + AggregationBuilder builder = aggregationBuilder.getLeft().getFirst(); + if (builder instanceof CompositeAggregationBuilder compositeAggBuilder) { + if (size < compositeAggBuilder.size()) { + compositeAggBuilder.size(size); + return true; + } else { + return false; + } + } + if (builder instanceof TermsAggregationBuilder termsAggBuilder) { + if (size < termsAggBuilder.size()) { + termsAggBuilder.size(size); + return true; + } else { + return false; + } + } + if (builder instanceof MultiTermsAggregationBuilder multiTermsAggBuilder) { + if (size < multiTermsAggBuilder.size()) { + multiTermsAggBuilder.size(size); + return true; + } else { + return false; + } + } + // now we only have Composite, Terms and MultiTerms bucket aggregations, + // add code here when we could support more in the future. + if (builder instanceof ValuesSourceAggregationBuilder.LeafOnly) { + // Note: all metric aggregations will be treated as pushed since it generates only one row. + return true; + } + throw new OpenSearchRequestBuilder.PushDownUnSupportedException( + "Unknown aggregation builder " + builder.getClass().getSimpleName()); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AggregationBuilderAction.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AggregationBuilderAction.java new file mode 100644 index 00000000000..cd3e84bf7cf --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AggregationBuilderAction.java @@ -0,0 +1,20 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.scan.context; + +/** A lambda action to apply on the {@link AggPushDownAction} */ +public interface AggregationBuilderAction extends AbstractAction { + /** + * Apply the action on the target {@link AggPushDownAction} and add the operation to the context + * + * @param context the context to add the operation to + * @param operation the operation to add to the context + */ + default void transform(PushDownContext context, PushDownOperation operation) { + apply(context.getAggPushDownAction()); + context.getOperationsForAgg().add(operation); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/FilterDigest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/FilterDigest.java new file mode 100644 index 00000000000..30c8d01feb1 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/FilterDigest.java @@ -0,0 +1,15 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.scan.context; + +import org.apache.calcite.rex.RexNode; + +public record FilterDigest(int scriptCount, RexNode condition) { + @Override + public String toString() { + return condition.toString(); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/LimitDigest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/LimitDigest.java new file mode 100644 index 00000000000..342cd33d4b8 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/LimitDigest.java @@ -0,0 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.scan.context; + +public record LimitDigest(int limit, int offset) { + @Override + public String toString() { + return offset == 0 ? String.valueOf(limit) : "[" + limit + " from " + offset + "]"; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/OSRequestBuilderAction.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/OSRequestBuilderAction.java new file mode 100644 index 00000000000..bba33883b49 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/OSRequestBuilderAction.java @@ -0,0 +1,23 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.scan.context; + +import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder; + +/** A lambda action to apply on the {@link OpenSearchRequestBuilder} */ +public interface OSRequestBuilderAction extends AbstractAction { + /** + * Apply the action on the target {@link OpenSearchRequestBuilder} and add the operation to the + * context + * + * @param context the context to add the operation to + * @param operation the operation to add to the context + */ + default void transform(PushDownContext context, PushDownOperation operation) { + apply(context.getRequestBuilder()); + context.getOperationsForRequestBuilder().add(operation); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/PushDownContext.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/PushDownContext.java new file mode 100644 index 00000000000..1b50a2a8751 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/PushDownContext.java @@ -0,0 +1,139 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.scan.context; + +import com.google.common.collect.Iterators; +import java.util.AbstractCollection; +import java.util.ArrayDeque; +import java.util.Collections; +import java.util.Iterator; +import lombok.Getter; +import org.jetbrains.annotations.NotNull; +import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder; +import org.opensearch.sql.opensearch.storage.OpenSearchIndex; + +/** Push down context is used to store all the push down operations that are applied to the query */ +@Getter +public class PushDownContext extends AbstractCollection { + private final OpenSearchIndex osIndex; + private final OpenSearchRequestBuilder requestBuilder; + private ArrayDeque operationsForRequestBuilder; + + private boolean isAggregatePushed = false; + private AggPushDownAction aggPushDownAction; + private ArrayDeque operationsForAgg; + + private boolean isLimitPushed = false; + private boolean isProjectPushed = false; + private boolean isMeasureOrderPushed = false; + private boolean isSortPushed = false; + private boolean isTopKPushed = false; + private boolean isRareTopPushed = false; + + public PushDownContext(OpenSearchIndex osIndex) { + this.osIndex = osIndex; + this.requestBuilder = osIndex.createRequestBuilder(); + } + + @Override + public PushDownContext clone() { + PushDownContext newContext = new PushDownContext(osIndex); + newContext.addAll(this); + return newContext; + } + + /** + * Create a new {@link PushDownContext} without the collation action. + * + * @return A new push-down context without the collation action. + */ + public PushDownContext cloneWithoutSort() { + PushDownContext newContext = new PushDownContext(osIndex); + for (PushDownOperation action : this) { + if (action.type() != PushDownType.SORT) { + newContext.add(action); + } + } + return newContext; + } + + @NotNull + @Override + public Iterator iterator() { + if (operationsForRequestBuilder == null) { + return Collections.emptyIterator(); + } else if (operationsForAgg == null) { + return operationsForRequestBuilder.iterator(); + } else { + return Iterators.concat(operationsForRequestBuilder.iterator(), operationsForAgg.iterator()); + } + } + + @Override + public int size() { + return (operationsForRequestBuilder == null ? 0 : operationsForRequestBuilder.size()) + + (operationsForAgg == null ? 0 : operationsForAgg.size()); + } + + ArrayDeque getOperationsForRequestBuilder() { + if (operationsForRequestBuilder == null) { + this.operationsForRequestBuilder = new ArrayDeque<>(); + } + return operationsForRequestBuilder; + } + + ArrayDeque getOperationsForAgg() { + if (operationsForAgg == null) { + this.operationsForAgg = new ArrayDeque<>(); + } + return operationsForAgg; + } + + @Override + public boolean add(PushDownOperation operation) { + if (operation.type() == PushDownType.AGGREGATION) { + isAggregatePushed = true; + this.aggPushDownAction = (AggPushDownAction) operation.action(); + } + if (operation.type() == PushDownType.LIMIT) { + isLimitPushed = true; + if (isSortPushed || isMeasureOrderPushed) { + isTopKPushed = true; + } + } + if (operation.type() == PushDownType.PROJECT) { + isProjectPushed = true; + } + if (operation.type() == PushDownType.SORT) { + isSortPushed = true; + } + if (operation.type() == PushDownType.SORT_AGG_METRICS) { + isMeasureOrderPushed = true; + } + if (operation.type() == PushDownType.RARE_TOP) { + isRareTopPushed = true; + } + operation.action().transform(this, operation); + return true; + } + + public void add(PushDownType type, Object digest, AbstractAction action) { + add(new PushDownOperation(type, digest, action)); + } + + public boolean containsDigest(Object digest) { + return this.stream().anyMatch(action -> action.digest().equals(digest)); + } + + public OpenSearchRequestBuilder createRequestBuilder() { + OpenSearchRequestBuilder newRequestBuilder = osIndex.createRequestBuilder(); + if (operationsForRequestBuilder != null) { + operationsForRequestBuilder.forEach( + operation -> ((OSRequestBuilderAction) operation.action()).apply(newRequestBuilder)); + } + return newRequestBuilder; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/PushDownOperation.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/PushDownOperation.java new file mode 100644 index 00000000000..c5779564369 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/PushDownOperation.java @@ -0,0 +1,19 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.scan.context; + +/** + * Represents a push down operation that can be applied to an OpenSearchRequestBuilder. + * + * @param type PushDownType enum + * @param digest the digest of the pushed down operator + * @param action the lambda action to apply on the OpenSearchRequestBuilder + */ +public record PushDownOperation(PushDownType type, Object digest, AbstractAction action) { + public String toString() { + return type + "->" + digest; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/PushDownType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/PushDownType.java new file mode 100644 index 00000000000..ddb0a3d7e66 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/PushDownType.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.scan.context; + +/** Push down types. */ +public enum PushDownType { + FILTER, + PROJECT, + AGGREGATION, + SORT, + LIMIT, + SCRIPT, + COLLAPSE, + SORT_AGG_METRICS, // convert composite aggregate to terms or multi-terms bucket aggregate + RARE_TOP, // convert composite aggregate to nested aggregate + // HIGHLIGHT, + // NESTED +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/RareTopDigest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/RareTopDigest.java new file mode 100644 index 00000000000..5ddd90af8ef --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/RareTopDigest.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.scan.context; + +import static org.apache.calcite.rel.RelFieldCollation.Direction.ASCENDING; + +import java.util.List; +import joptsimple.internal.Strings; +import org.apache.calcite.rel.RelFieldCollation; + +public record RareTopDigest( + String target, List byList, Integer number, RelFieldCollation.Direction direction) { + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append(direction == ASCENDING ? "rare" : "top"); + builder.append(" "); + builder.append(number); + builder.append(" "); + builder.append(target); + if (!byList.isEmpty()) { + builder.append(" by "); + builder.append(Strings.join(byList, ",")); + } + return builder.toString(); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilder.java index 408511fde3f..e11c5aa9728 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilder.java @@ -29,7 +29,7 @@ import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.aggregation.NamedAggregator; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; -import org.opensearch.sql.opensearch.response.agg.CompositeAggregationParser; +import org.opensearch.sql.opensearch.response.agg.BucketAggregationParser; import org.opensearch.sql.opensearch.response.agg.MetricParser; import org.opensearch.sql.opensearch.response.agg.NoBucketAggregationParser; import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; @@ -95,7 +95,7 @@ public AggregationQueryBuilder(ExpressionSerializer serializer) { bucketNullable)) .subAggregations(metrics.getLeft()) .size(AGGREGATION_BUCKET_SIZE)), - new CompositeAggregationParser(metrics.getRight())); + new BucketAggregationParser(metrics.getRight())); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java index 9cb3f9824b8..0f523d65341 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java @@ -215,7 +215,7 @@ private Pair make( String name, MetricParser parser) { String fieldName = ((ReferenceExpression) expression).getAttr(); - builder.fetchSource(fieldName, null); + builder.fetchField(fieldName); builder.size(size.valueOf().integerValue()); builder.from(0); if (condition != null) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializer.java index 12ae05ea735..fb751c8a72c 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializer.java @@ -12,7 +12,9 @@ import java.io.ByteArrayOutputStream; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; +import java.io.Serializable; import java.util.Base64; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import lombok.Getter; @@ -26,9 +28,12 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.util.SqlOperatorTables; import org.apache.calcite.util.JsonBuilder; +import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.calcite.CalcitePlanContext; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.function.PPLBuiltinOperators; +import org.opensearch.sql.opensearch.executor.OpenSearchExecutionEngine; +import org.opensearch.sql.opensearch.util.OpenSearchRelOptUtil; /** * A serializer that (de-)serializes Calcite RexNode, RelDataType and OpenSearch field mapping. @@ -36,7 +41,7 @@ *

This serializer: *

  • Uses Calcite's RelJson class to convert RexNode and RelDataType to/from JSON string *
  • Manages required OpenSearch field mapping information Note: OpenSearch ExprType subclasses - * implement {@link java.io.Serializable} and are handled through standard Java serialization. + * implement {@link Serializable} and are handled through standard Java serialization. */ @Getter public class RelJsonSerializer { @@ -49,13 +54,7 @@ public class RelJsonSerializer { private static final ObjectMapper mapper = new ObjectMapper(); private static final TypeReference> TYPE_REF = new TypeReference<>() {}; - private static final SqlOperatorTable pplSqlOperatorTable = - SqlOperatorTables.chain( - PPLBuiltinOperators.instance(), - SqlStdOperatorTable.instance(), - // Add a list of necessary SqlLibrary if needed - SqlLibraryOperatorTableFactory.INSTANCE.getOperatorTable( - SqlLibrary.MYSQL, SqlLibrary.BIG_QUERY, SqlLibrary.SPARK, SqlLibrary.POSTGRESQL)); + private static volatile SqlOperatorTable pplSqlOperatorTable; static { mapper.configure(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS, true); @@ -65,6 +64,27 @@ public RelJsonSerializer(RelOptCluster cluster) { this.cluster = cluster; } + private static SqlOperatorTable getPplSqlOperatorTable() { + if (pplSqlOperatorTable == null) { + synchronized (RelJsonSerializer.class) { + if (pplSqlOperatorTable == null) { + pplSqlOperatorTable = + SqlOperatorTables.chain( + PPLBuiltinOperators.instance(), + SqlStdOperatorTable.instance(), + OpenSearchExecutionEngine.OperatorTable.instance(), + // Add a list of necessary SqlLibrary if needed + SqlLibraryOperatorTableFactory.INSTANCE.getOperatorTable( + SqlLibrary.MYSQL, + SqlLibrary.BIG_QUERY, + SqlLibrary.SPARK, + SqlLibrary.POSTGRESQL)); + } + } + } + return pplSqlOperatorTable; + } + /** * Serializes Calcite expressions and field types into a map object string. * @@ -79,16 +99,23 @@ public RelJsonSerializer(RelOptCluster cluster) { * @return serialized string of map structure for inputs */ public String serialize(RexNode rexNode, RelDataType rowType, Map fieldTypes) { + // Extract necessary fields and remap expression input indices for original RexNode + Pair remappedRexInfo = + OpenSearchRelOptUtil.getRemappedRexAndType(rexNode, rowType); + Map filteredFieldTypes = new HashMap<>(); + for (String fieldName : remappedRexInfo.getValue().getFieldNames()) { + filteredFieldTypes.put(fieldName, fieldTypes.get(fieldName)); + } try { // Serialize RexNode and RelDataType by JSON JsonBuilder jsonBuilder = new JsonBuilder(); RelJson relJson = ExtendedRelJson.create(jsonBuilder); - String rexNodeJson = jsonBuilder.toJsonString(relJson.toJson(rexNode)); - Object rowTypeJsonObj = relJson.toJson(rowType); + String rexNodeJson = jsonBuilder.toJsonString(relJson.toJson(remappedRexInfo.getKey())); + Object rowTypeJsonObj = relJson.toJson(remappedRexInfo.getValue()); String rowTypeJson = jsonBuilder.toJsonString(rowTypeJsonObj); // Construct envelope of serializable objects Map envelope = - Map.of(EXPR, rexNodeJson, FIELD_TYPES, fieldTypes, ROW_TYPE, rowTypeJson); + Map.of(EXPR, rexNodeJson, FIELD_TYPES, filteredFieldTypes, ROW_TYPE, rowTypeJson); // Write bytes of all serializable contents ByteArrayOutputStream output = new ByteArrayOutputStream(); @@ -99,7 +126,8 @@ public String serialize(RexNode rexNode, RelDataType rowType, Map deserialize(String struct) { Map rowTypeMap = mapper.readValue((String) objectMap.get(ROW_TYPE), TYPE_REF); RelDataType rowType = relJson.toType(cluster.getTypeFactory(), rowTypeMap); OpenSearchRelInputTranslator inputTranslator = new OpenSearchRelInputTranslator(rowType); - relJson = relJson.withInputTranslator(inputTranslator).withOperatorTable(pplSqlOperatorTable); + relJson = + relJson.withInputTranslator(inputTranslator).withOperatorTable(getPplSqlOperatorTable()); Map exprMap = mapper.readValue((String) objectMap.get(EXPR), TYPE_REF); RexNode rexNode = relJson.toRex(cluster, exprMap); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java index bcbc5bec009..012ceec8c13 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java @@ -14,7 +14,7 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.type.RelDataType; -import org.opensearch.sql.opensearch.planner.physical.EnumerableSystemIndexScanRule; +import org.opensearch.sql.opensearch.planner.rules.EnumerableSystemIndexScanRule; /** The logical relational operator representing a scan of an OpenSearchSystemIndex type. */ public class CalciteLogicalSystemIndexScan extends AbstractCalciteSystemIndexScan { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java index 86b09911908..ab3743aeaf4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java @@ -6,20 +6,61 @@ package org.opensearch.sql.opensearch.util; import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.HashSet; +import java.util.List; import java.util.Optional; +import java.util.Set; import lombok.experimental.UtilityClass; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rel.type.RelDataTypeFieldImpl; +import org.apache.calcite.rex.RexBiVisitorImpl; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.type.SqlTypeUtil; +import org.apache.calcite.util.mapping.Mapping; +import org.apache.calcite.util.mapping.Mappings; import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; @UtilityClass public class OpenSearchRelOptUtil { + private static final RemapIndexBiVisitor remapIndexBiVisitor = new RemapIndexBiVisitor(true); + + /** + * For pushed down RexNode, the input schema doesn't need to be the same with scan output schema + * because the input values are read from ScriptDocValues or source by field name. It doesn't + * matter what the actual index is. Current serialization will serialize map of field name and + * field ExprType, which could be a long serialized string. Use this method to narrow down input + * rowType and rewrite RexNode's input references. After that, we can leverage the fewer columns + * in the rowType to serialize least required field types. + * + * @param rexNode original RexNode to be pushed down + * @param inputRowType original input rowType of RexNode + * @return rewritten pair of RexNode and RelDataType + */ + public static Pair getRemappedRexAndType( + final RexNode rexNode, final RelDataType inputRowType) { + final BitSet seenOldIndex = new BitSet(); + final List newMappings = new ArrayList<>(); + rexNode.accept(remapIndexBiVisitor, Pair.of(seenOldIndex, newMappings)); + final List inputFieldList = inputRowType.getFieldList(); + final RelDataTypeFactory.Builder builder = OpenSearchTypeFactory.TYPE_FACTORY.builder(); + for (Integer oldIdx : newMappings) { + builder.add(inputFieldList.get(oldIdx)); + } + final Mapping mapping = Mappings.target(newMappings, inputRowType.getFieldCount()); + final RexNode newMappedRex = RexUtil.apply(mapping, rexNode); + return Pair.of(newMappedRex, builder.build()); + } /** * Given an input Calcite RexNode, find the single input field with equivalent collation @@ -148,4 +189,89 @@ private static boolean isOrderPreservingCast(RelDataType src, RelDataType dst) { return false; } + + private static class RemapIndexBiVisitor + extends RexBiVisitorImpl>> { + protected RemapIndexBiVisitor(boolean deep) { + super(deep); + } + + @Override + public Void visitInputRef(RexInputRef inputRef, Pair> args) { + final BitSet seenOldIndex = args.getLeft(); + final List newMappings = args.getRight(); + final int oldIdx = inputRef.getIndex(); + if (!seenOldIndex.get(oldIdx)) { + seenOldIndex.set(oldIdx); + newMappings.add(oldIdx); + } + return null; + } + } + + /** + * Replace dot in field name with underscore, since Calcite has bug in codegen if a field name + * contains dot. + * + *

    Fields replacement examples: + * + *

    a_b, a.b -> a_b, a_b0 + * + *

    a_b, a_b0, a.b -> a_b, a_b0, a_b1 + * + *

    a_b, a_b1, a.b -> a_b, a_b1, a_b0 + * + *

    a_b0, a.b0, a.b1 -> a_b0, a_b00, a_b1 + * + * @param rowType RowType + * @return RowType with field name replaced + */ + public RelDataType replaceDot(RelDataTypeFactory typeFactory, RelDataType rowType) { + final RelDataTypeFactory.Builder builder = typeFactory.builder(); + final List fieldList = rowType.getFieldList(); + List originalNames = new ArrayList<>(); + for (RelDataTypeField field : fieldList) { + originalNames.add(field.getName()); + } + List resolvedNames = OpenSearchRelOptUtil.resolveColumnNameConflicts(originalNames); + for (int i = 0; i < fieldList.size(); i++) { + RelDataTypeField field = fieldList.get(i); + builder.add( + new RelDataTypeFieldImpl(resolvedNames.get(i), field.getIndex(), field.getType())); + } + return builder.build(); + } + + public static List resolveColumnNameConflicts(List originalNames) { + List result = new ArrayList<>(originalNames); + Set usedNames = new HashSet<>(originalNames); + for (int i = 0; i < originalNames.size(); i++) { + String originalName = originalNames.get(i); + if (originalName.contains(".")) { + String baseName = originalName.replace('.', '_'); + String newName = generateUniqueName(baseName, usedNames); + result.set(i, newName); + usedNames.add(newName); + } + } + return result; + } + + private static String generateUniqueName(String baseName, Set usedNames) { + if (!usedNames.contains(baseName)) { + return baseName; + } + String candidate = baseName + "0"; + if (!usedNames.contains(candidate)) { + return candidate; + } + int suffix = 1; + while (true) { + candidate = baseName + suffix; + if (!usedNames.contains(candidate)) { + return candidate; + } + suffix++; + } + } } diff --git a/opensearch/src/main/resources/opensearch_custom_functions.yaml b/opensearch/src/main/resources/opensearch_custom_functions.yaml new file mode 100644 index 00000000000..81a0ff1ecfb --- /dev/null +++ b/opensearch/src/main/resources/opensearch_custom_functions.yaml @@ -0,0 +1,23 @@ +%YAML 1.2 +--- +urn: extension:org.opensearch:custom_functions +scalar_functions: + - name: date_format + description: Format a timestamp using a format string + impls: + - args: + - name: timestamp + value: precision_timestamp

    + - name: format + value: string + return: string + - name: date_part + description: Extract a part from a timestamp + impls: + - args: + - name: part + value: string + - name: timestamp + value: precision_timestamp

    + return: i64 + diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java index 948ac4854c0..9b6da17567e 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java @@ -187,7 +187,7 @@ void get_index_mappings() throws IOException { () -> assertEquals(OpenSearchTextType.of(MappingType.Text), parsedTypes.get("employer")), // `employer` is a `text` with `fields` () -> assertTrue(((OpenSearchTextType) parsedTypes.get("employer")).getFields().size() > 0), - () -> assertEquals("TEXT", mapping.get("employer_alias").legacyTypeName()), + () -> assertEquals("TEXT", parsedTypes.get("employer_alias").legacyTypeName()), () -> assertEquals( new OpenSearchAliasType("employer", OpenSearchTextType.of(MappingType.Text)), diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java index 6be02c9d6f1..88a70c08b94 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java @@ -191,7 +191,7 @@ void get_index_mappings() throws IOException { () -> assertEquals(OpenSearchTextType.of(MappingType.Text), parsedTypes.get("employer")), // `employer` is a `text` with `fields` () -> assertTrue(((OpenSearchTextType) parsedTypes.get("employer")).getFields().size() > 0), - () -> assertEquals("TEXT", mapping.get("employer_alias").legacyTypeName()), + () -> assertEquals("TEXT", parsedTypes.get("employer_alias").legacyTypeName()), () -> assertEquals( new OpenSearchAliasType("employer", OpenSearchTextType.of(MappingType.Text)), diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java index b85716a44db..40985130c52 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java @@ -505,9 +505,14 @@ public void test_parseMapping_on_AliasType() { "col0", Map.of("type", "alias", "path", "col1"), "col1", Map.of("type", "text"), "col2", Map.of("type", "alias", "path", "col3")); - IllegalStateException exception = - assertThrows( - IllegalStateException.class, () -> OpenSearchDataType.parseMapping(indexMapping2)); - assertEquals("Cannot find the path [col3] for alias type field [col2]", exception.getMessage()); + assertEquals( + Map.of( + "col0", + new OpenSearchAliasType("col1", textType), + "col1", + textType, + "col2", + new OpenSearchAliasType("col1", OpenSearchDataType.of(MappingType.Invalid))), + OpenSearchDataType.parseMapping(indexMapping2)); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java index e8588fa778c..32ba07d4d53 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java @@ -972,8 +972,8 @@ public void constructUnsupportedTypeThrowException() { @Test // aggregation adds info about new columns to the factory, - // it is accepted without overwriting existing data. - public void factoryMappingsAreExtendableWithoutOverWrite() + // it will overwrite existing type to fix https://github.com/opensearch-project/sql/issues/4115 + public void factoryMappingsAreExtendableWithOverWrite() throws NoSuchFieldException, IllegalAccessException { var factory = new OpenSearchExprValueFactory(Map.of("value", OpenSearchDataType.of(INTEGER)), true); @@ -990,7 +990,7 @@ public void factoryMappingsAreExtendableWithoutOverWrite() () -> assertEquals(2, mapping.size()), () -> assertTrue(mapping.containsKey("value")), () -> assertTrue(mapping.containsKey("agg")), - () -> assertEquals(OpenSearchDataType.of(INTEGER), mapping.get("value")), + () -> assertEquals(OpenSearchDataType.of(DOUBLE), mapping.get("value")), () -> assertEquals(OpenSearchDataType.of(DATE), mapping.get("agg"))); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java index b3a1d766d8b..f07ec4139dc 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java @@ -46,7 +46,7 @@ import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType.MappingType; import org.opensearch.sql.opensearch.request.AggregateAnalyzer.ExpressionNotAnalyzableException; -import org.opensearch.sql.opensearch.response.agg.CompositeAggregationParser; +import org.opensearch.sql.opensearch.response.agg.BucketAggregationParser; import org.opensearch.sql.opensearch.response.agg.FilterParser; import org.opensearch.sql.opensearch.response.agg.MetricParserHelper; import org.opensearch.sql.opensearch.response.agg.NoBucketAggregationParser; @@ -56,7 +56,7 @@ import org.opensearch.sql.opensearch.response.agg.TopHitsParser; class AggregateAnalyzerTest { - + private static final int BUCKET_SIZE = 1000; private final RelDataTypeFactory typeFactory = new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT); private final List schema = List.of("a", "b", "c", "d"); private final RelDataType rowType = @@ -152,8 +152,10 @@ void analyze_aggCall_simple() throws ExpressionNotAnalyzableException { createMockAggregate( List.of(countCall, avgCall, sumCall, minCall, maxCall), ImmutableBitSet.of()); Project project = createMockProject(List.of(0)); + AggregateAnalyzer.AggregateBuilderHelper helper = + new AggregateAnalyzer.AggregateBuilderHelper(rowType, fieldTypes, null, true, BUCKET_SIZE); Pair, OpenSearchAggregationResponseParser> result = - AggregateAnalyzer.analyze(aggregate, project, rowType, fieldTypes, outputFields, null); + AggregateAnalyzer.analyze(aggregate, project, outputFields, helper); assertEquals( "[{\"cnt\":{\"value_count\":{\"field\":\"_index\"}}}," + " {\"avg\":{\"avg\":{\"field\":\"a\"}}}," @@ -233,8 +235,10 @@ void analyze_aggCall_extended() throws ExpressionNotAnalyzableException { createMockAggregate( List.of(varSampCall, varPopCall, stddevSampCall, stddevPopCall), ImmutableBitSet.of()); Project project = createMockProject(List.of(0)); + AggregateAnalyzer.AggregateBuilderHelper helper = + new AggregateAnalyzer.AggregateBuilderHelper(rowType, fieldTypes, null, true, BUCKET_SIZE); Pair, OpenSearchAggregationResponseParser> result = - AggregateAnalyzer.analyze(aggregate, project, rowType, fieldTypes, outputFields, null); + AggregateAnalyzer.analyze(aggregate, project, outputFields, helper); assertEquals( "[{\"var_samp\":{\"extended_stats\":{\"field\":\"a\",\"sigma\":2.0}}}," + " {\"var_pop\":{\"extended_stats\":{\"field\":\"a\",\"sigma\":2.0}}}," @@ -272,17 +276,20 @@ void analyze_groupBy() throws ExpressionNotAnalyzableException { List outputFields = List.of("a", "b", "cnt"); Aggregate aggregate = createMockAggregate(List.of(aggCall), ImmutableBitSet.of(0, 1)); Project project = createMockProject(List.of(0, 1)); + AggregateAnalyzer.AggregateBuilderHelper helper = + new AggregateAnalyzer.AggregateBuilderHelper(rowType, fieldTypes, null, true, BUCKET_SIZE); Pair, OpenSearchAggregationResponseParser> result = - AggregateAnalyzer.analyze(aggregate, project, rowType, fieldTypes, outputFields, null); + AggregateAnalyzer.analyze(aggregate, project, outputFields, helper); assertEquals( "[{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[" + "{\"a\":{\"terms\":{\"field\":\"a\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}," + "{\"b\":{\"terms\":{\"field\":\"b.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}]", result.getLeft().toString()); - assertInstanceOf(CompositeAggregationParser.class, result.getRight()); + assertInstanceOf(BucketAggregationParser.class, result.getRight()); + assertInstanceOf(BucketAggregationParser.class, result.getRight()); MetricParserHelper metricsParser = - ((CompositeAggregationParser) result.getRight()).getMetricsParser(); + ((BucketAggregationParser) result.getRight()).getMetricsParser(); assertEquals(1, metricsParser.getMetricParserMap().size()); metricsParser .getMetricParserMap() @@ -310,12 +317,12 @@ void analyze_aggCall_TextWithoutKeyword() { "sum"); Aggregate aggregate = createMockAggregate(List.of(aggCall), ImmutableBitSet.of()); Project project = createMockProject(List.of(2)); + AggregateAnalyzer.AggregateBuilderHelper helper = + new AggregateAnalyzer.AggregateBuilderHelper(rowType, fieldTypes, null, true, BUCKET_SIZE); ExpressionNotAnalyzableException exception = assertThrows( ExpressionNotAnalyzableException.class, - () -> - AggregateAnalyzer.analyze( - aggregate, project, rowType, fieldTypes, List.of("sum"), null)); + () -> AggregateAnalyzer.analyze(aggregate, project, List.of("sum"), helper)); assertEquals("[field] must not be null: [sum]", exception.getCause().getMessage()); } @@ -337,12 +344,12 @@ void analyze_groupBy_TextWithoutKeyword() { List outputFields = List.of("c", "cnt"); Aggregate aggregate = createMockAggregate(List.of(aggCall), ImmutableBitSet.of(0)); Project project = createMockProject(List.of(2)); + AggregateAnalyzer.AggregateBuilderHelper helper = + new AggregateAnalyzer.AggregateBuilderHelper(rowType, fieldTypes, null, true, BUCKET_SIZE); ExpressionNotAnalyzableException exception = assertThrows( ExpressionNotAnalyzableException.class, - () -> - AggregateAnalyzer.analyze( - aggregate, project, rowType, fieldTypes, outputFields, null)); + () -> AggregateAnalyzer.analyze(aggregate, project, outputFields, helper)); assertEquals("[field] must not be null", exception.getCause().getMessage()); } @@ -591,8 +598,11 @@ private Project createMockProject(List refIndex) { when(ref.getType()).thenReturn(typeFactory.createSqlType(SqlTypeName.INTEGER)); rexNodes.add(ref); } + List> namedProjects = + rexNodes.stream().map(n -> org.apache.calcite.util.Pair.of(n, n.toString())).toList(); when(project.getProjects()).thenReturn(rexNodes); when(project.getRowType()).thenReturn(rowType); + when(project.getNamedProjects()).thenReturn(namedProjects); return project; } @@ -685,9 +695,11 @@ void verify() throws ExpressionNotAnalyzableException { if (agg.getInput(0) instanceof Project) { project = (Project) agg.getInput(0); } + AggregateAnalyzer.AggregateBuilderHelper helper = + new AggregateAnalyzer.AggregateBuilderHelper( + rowType, fieldTypes, agg.getCluster(), true, BUCKET_SIZE); Pair, OpenSearchAggregationResponseParser> result = - AggregateAnalyzer.analyze( - agg, project, rowType, fieldTypes, outputFields, agg.getCluster()); + AggregateAnalyzer.analyze(agg, project, outputFields, helper); if (expectedDsl != null) { assertEquals(expectedDsl, result.getLeft().toString()); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/CaseRangeAnalyzerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/CaseRangeAnalyzerTest.java new file mode 100644 index 00000000000..505db011f7b --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/CaseRangeAnalyzerTest.java @@ -0,0 +1,844 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.request; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.google.common.collect.Range; +import com.google.common.collect.TreeRangeSet; +import java.math.BigDecimal; +import java.util.Arrays; +import java.util.Optional; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeSystem; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUnknownAs; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeFactoryImpl; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.util.Sarg; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.opensearch.search.aggregations.bucket.range.RangeAggregationBuilder; + +class CaseRangeAnalyzerTest { + + private RelDataTypeFactory typeFactory; + private RexBuilder rexBuilder; + private RelDataType rowType; + private RexInputRef fieldRef; + + @BeforeEach + void setUp() { + typeFactory = new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT); + rexBuilder = new RexBuilder(typeFactory); + + // Create a row type with fields: age (INTEGER), name (VARCHAR) + rowType = + typeFactory + .builder() + .add("age", SqlTypeName.INTEGER) + .add("name", SqlTypeName.VARCHAR) + .build(); + + fieldRef = rexBuilder.makeInputRef(rowType.getFieldList().get(0).getType(), 0); // age field + } + + @Test + void testAnalyzeSimpleCaseExpression() { + // CASE + // WHEN age >= 18 THEN 'adult' + // WHEN age >= 13 THEN 'teen' + // ELSE 'child' + // END + + RexLiteral literal18 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(18)); + RexLiteral literal13 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(13)); + RexLiteral adultLiteral = rexBuilder.makeLiteral("adult"); + RexLiteral teenLiteral = rexBuilder.makeLiteral("teen"); + RexLiteral childLiteral = rexBuilder.makeLiteral("child"); + + RexCall condition1 = + (RexCall) + rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, fieldRef, literal18); + RexCall condition2 = + (RexCall) + rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, fieldRef, literal13); + + RexCall caseCall = + (RexCall) + rexBuilder.makeCall( + SqlStdOperatorTable.CASE, + Arrays.asList(condition1, adultLiteral, condition2, teenLiteral, childLiteral)); + + CaseRangeAnalyzer analyzer = CaseRangeAnalyzer.create("age_ranges", rowType); + Optional result = analyzer.analyze(caseCall); + + assertTrue(result.isPresent()); + RangeAggregationBuilder builder = result.get(); + assertEquals("age_ranges", builder.getName()); + assertEquals("age", builder.field()); + + String expectedJson = + """ + { + "age_ranges" : { + "range" : { + "field" : "age", + "ranges" : [ + { + "key" : "adult", + "from" : 18.0 + }, + { + "key" : "teen", + "from" : 13.0, + "to" : 18.0 + }, + { + "key" : "child", + "to" : 13.0 + } + ], + "keyed" : true + } + } + }"""; + + assertEquals(normalizeJson(expectedJson), normalizeJson(builder.toString())); + } + + @Test + void testAnalyzeLessThanComparison() { + // CASE WHEN age < 18 THEN 'minor' ELSE 'adult' END + + RexLiteral literal18 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(18)); + RexLiteral minorLiteral = rexBuilder.makeLiteral("minor"); + RexLiteral adultLiteral = rexBuilder.makeLiteral("adult"); + + RexCall condition = + (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, fieldRef, literal18); + + RexCall caseCall = + (RexCall) + rexBuilder.makeCall( + SqlStdOperatorTable.CASE, Arrays.asList(condition, minorLiteral, adultLiteral)); + + CaseRangeAnalyzer analyzer = CaseRangeAnalyzer.create("age_check", rowType); + Optional result = analyzer.analyze(caseCall); + + assertTrue(result.isPresent()); + RangeAggregationBuilder builder = result.get(); + + String expectedJson = + """ + { + "age_check" : { + "range" : { + "field" : "age", + "ranges" : [ + { + "key" : "minor", + "to" : 18.0 + }, + { + "key" : "adult", + "from" : 18.0 + } + ], + "keyed" : true + } + } + }"""; + + assertEquals(normalizeJson(expectedJson), normalizeJson(builder.toString())); + } + + @Test + void testAnalyzeWithSearchCondition() { + // Create a SEARCH condition (Sarg-based range) + TreeRangeSet rangeSet = TreeRangeSet.create(); + rangeSet.add(Range.closedOpen(BigDecimal.valueOf(18), BigDecimal.valueOf(65))); + + Sarg sarg = Sarg.of(RexUnknownAs.UNKNOWN, rangeSet); + RexNode sargLiteral = + rexBuilder.makeSearchArgumentLiteral(sarg, typeFactory.createSqlType(SqlTypeName.DECIMAL)); + + RexCall searchCall = + (RexCall) + rexBuilder.makeCall(SqlStdOperatorTable.SEARCH, Arrays.asList(fieldRef, sargLiteral)); + + RexLiteral workingLiteral = rexBuilder.makeLiteral("working_age"); + RexLiteral otherLiteral = rexBuilder.makeLiteral("other"); + + RexCall caseCall = + (RexCall) + rexBuilder.makeCall( + SqlStdOperatorTable.CASE, Arrays.asList(searchCall, workingLiteral, otherLiteral)); + + CaseRangeAnalyzer analyzer = CaseRangeAnalyzer.create("age_groups", rowType); + Optional result = analyzer.analyze(caseCall); + + assertTrue(result.isPresent()); + RangeAggregationBuilder builder = result.get(); + + String expectedJson = + """ + { + "age_groups" : { + "range" : { + "field" : "age", + "ranges" : [ + { + "key" : "working_age", + "from" : 18.0, + "to" : 65.0 + }, + { + "key" : "other", + "to" : 18.0 + }, + { + "key" : "other", + "from" : 65.0 + } + ], + "keyed" : true + } + } + }"""; + + assertEquals(normalizeJson(expectedJson), normalizeJson(builder.toString())); + } + + @Test + void testAnalyzeWithNullElse() { + // CASE WHEN age >= 18 THEN 'adult' ELSE NULL END + + RexLiteral literal18 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(18)); + RexLiteral adultLiteral = rexBuilder.makeLiteral("adult"); + RexLiteral nullLiteral = + rexBuilder.makeNullLiteral(typeFactory.createSqlType(SqlTypeName.VARCHAR)); + + RexCall condition = + (RexCall) + rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, fieldRef, literal18); + + RexCall caseCall = + (RexCall) + rexBuilder.makeCall( + SqlStdOperatorTable.CASE, Arrays.asList(condition, adultLiteral, nullLiteral)); + + CaseRangeAnalyzer analyzer = CaseRangeAnalyzer.create("age_check", rowType); + Optional result = analyzer.analyze(caseCall); + + assertTrue(result.isPresent()); + RangeAggregationBuilder builder = result.get(); + // Should use DEFAULT_ELSE_KEY for null else clause + + String expectedJson = + """ + { + "age_check" : { + "range" : { + "field" : "age", + "ranges" : [ + { + "key" : "adult", + "from" : 18.0 + }, + { + "key" : "null", + "to" : 18.0 + } + ], + "keyed" : true + } + } + }"""; + + assertEquals(normalizeJson(expectedJson), normalizeJson(builder.toString())); + } + + @Test + void testAnalyzeWithNonLiteralResultShouldNotSucceed() { + // CASE WHEN age >= 18 THEN age ELSE 0 END (non-literal result) + + RexLiteral literal18 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(18)); + RexLiteral zeroLiteral = rexBuilder.makeExactLiteral(BigDecimal.valueOf(0)); + + RexCall condition = + (RexCall) + rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, fieldRef, literal18); + + RexCall caseCall = + (RexCall) + rexBuilder.makeCall( + SqlStdOperatorTable.CASE, + Arrays.asList(condition, fieldRef, zeroLiteral)); // fieldRef as result, not literal + + CaseRangeAnalyzer analyzer = CaseRangeAnalyzer.create("test", rowType); + Optional result = analyzer.analyze(caseCall); + + assertFalse(result.isPresent()); + } + + @Test + void testAnalyzeDifferentFieldsShouldReturnEmpty() { + // Test comparing different fields in conditions + RexInputRef nameFieldRef = rexBuilder.makeInputRef(rowType.getFieldList().get(1).getType(), 1); + + RexLiteral literal18 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(18)); + RexLiteral literalName = rexBuilder.makeLiteral("John"); + RexLiteral result1 = rexBuilder.makeLiteral("result1"); + RexLiteral result2 = rexBuilder.makeLiteral("result2"); + RexLiteral elseResult = rexBuilder.makeLiteral("else"); + + RexCall condition1 = + (RexCall) + rexBuilder.makeCall( + SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, fieldRef, literal18); // age >= 18 + RexCall condition2 = + (RexCall) + rexBuilder.makeCall( + SqlStdOperatorTable.EQUALS, nameFieldRef, literalName); // name = 'John' + + RexCall caseCall = + (RexCall) + rexBuilder.makeCall( + SqlStdOperatorTable.CASE, + Arrays.asList(condition1, result1, condition2, result2, elseResult)); + + CaseRangeAnalyzer analyzer = CaseRangeAnalyzer.create("test", rowType); + Optional result = analyzer.analyze(caseCall); + + assertFalse(result.isPresent()); + } + + @Test + void testAnalyzeWithAndConditionShouldReturnEmpty() { + // Test AND condition which should be unsupported + RexLiteral literal18 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(18)); + RexLiteral literal65 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(65)); + RexLiteral resultLiteral = rexBuilder.makeLiteral("working_age"); + RexLiteral elseLiteral = rexBuilder.makeLiteral("other"); + + RexCall condition1 = + (RexCall) + rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, fieldRef, literal18); + RexCall condition2 = + (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, fieldRef, literal65); + RexCall andCondition = + (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.AND, condition1, condition2); + + RexCall caseCall = + (RexCall) + rexBuilder.makeCall( + SqlStdOperatorTable.CASE, Arrays.asList(andCondition, resultLiteral, elseLiteral)); + + CaseRangeAnalyzer analyzer = CaseRangeAnalyzer.create("test", rowType); + Optional result = analyzer.analyze(caseCall); + + assertFalse(result.isPresent()); + } + + @Test + void testAnalyzeWithOrConditionShouldReturnEmpty() { + // Test OR condition which should be unsupported + RexLiteral literal18 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(18)); + RexLiteral literal65 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(65)); + RexLiteral resultLiteral = rexBuilder.makeLiteral("age_group"); + RexLiteral elseLiteral = rexBuilder.makeLiteral("other"); + + RexCall condition1 = + (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, fieldRef, literal18); + RexCall condition2 = + (RexCall) + rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, fieldRef, literal65); + RexCall orCondition = + (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.OR, condition1, condition2); + + RexCall caseCall = + (RexCall) + rexBuilder.makeCall( + SqlStdOperatorTable.CASE, Arrays.asList(orCondition, resultLiteral, elseLiteral)); + + CaseRangeAnalyzer analyzer = CaseRangeAnalyzer.create("test", rowType); + Optional result = analyzer.analyze(caseCall); + + assertFalse(result.isPresent()); + } + + @Test + void testAnalyzeWithUnsupportedComparison() { + // Test GREATER_THAN which should be converted to supported operations or fail + RexLiteral literal18 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(18)); + RexLiteral resultLiteral = rexBuilder.makeLiteral("adult"); + RexLiteral elseLiteral = rexBuilder.makeLiteral("minor"); + + RexCall condition = + (RexCall) + rexBuilder.makeCall( + SqlStdOperatorTable.GREATER_THAN, fieldRef, literal18); // This should fail + + RexCall caseCall = + (RexCall) + rexBuilder.makeCall( + SqlStdOperatorTable.CASE, Arrays.asList(condition, resultLiteral, elseLiteral)); + + CaseRangeAnalyzer analyzer = CaseRangeAnalyzer.create("test", rowType); + Optional result = analyzer.analyze(caseCall); + + assertFalse(result.isPresent()); + } + + @Test + void testAnalyzeWithReversedComparison() { + // Test literal on left side: 18 <= age (should be converted to age >= 18) + RexLiteral literal18 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(18)); + RexLiteral resultLiteral = rexBuilder.makeLiteral("adult"); + RexLiteral elseLiteral = rexBuilder.makeLiteral("minor"); + + RexCall condition = + (RexCall) + rexBuilder.makeCall( + SqlStdOperatorTable.LESS_THAN_OR_EQUAL, literal18, fieldRef); // 18 <= age + + RexCall caseCall = + (RexCall) + rexBuilder.makeCall( + SqlStdOperatorTable.CASE, Arrays.asList(condition, resultLiteral, elseLiteral)); + + CaseRangeAnalyzer analyzer = CaseRangeAnalyzer.create("reversed_test", rowType); + Optional result = analyzer.analyze(caseCall); + + assertTrue(result.isPresent()); + RangeAggregationBuilder builder = result.get(); + + String expectedJson = + """ + { + "reversed_test" : { + "range" : { + "field" : "age", + "ranges" : [ + { + "key" : "adult", + "from" : 18.0 + }, + { + "key" : "minor", + "to" : 18.0 + } + ], + "keyed" : true + } + } + }"""; + + assertEquals(normalizeJson(expectedJson), normalizeJson(builder.toString())); + } + + @Test + void testAnalyzeWithNullLiteralValue() { + // Test with null literal value that can't be converted to Double + RexLiteral nullLiteral = + rexBuilder.makeNullLiteral(typeFactory.createSqlType(SqlTypeName.INTEGER)); + RexLiteral resultLiteral = rexBuilder.makeLiteral("result"); + RexLiteral elseLiteral = rexBuilder.makeLiteral("else"); + + RexCall condition = + (RexCall) + rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, fieldRef, nullLiteral); + + RexCall caseCall = + (RexCall) + rexBuilder.makeCall( + SqlStdOperatorTable.CASE, Arrays.asList(condition, resultLiteral, elseLiteral)); + + CaseRangeAnalyzer analyzer = CaseRangeAnalyzer.create("test", rowType); + Optional result = analyzer.analyze(caseCall); + + assertFalse(result.isPresent()); + } + + @Test + void testSimpleCaseGeneratesExpectedDSL() { + // CASE WHEN age >= 18 THEN 'adult' ELSE 'minor' END + + RexLiteral literal18 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(18)); + RexLiteral adultLiteral = rexBuilder.makeLiteral("adult"); + RexLiteral minorLiteral = rexBuilder.makeLiteral("minor"); + + RexCall condition = + (RexCall) + rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, fieldRef, literal18); + + RexCall caseCall = + (RexCall) + rexBuilder.makeCall( + SqlStdOperatorTable.CASE, Arrays.asList(condition, adultLiteral, minorLiteral)); + + CaseRangeAnalyzer analyzer = CaseRangeAnalyzer.create("age_groups", rowType); + Optional result = analyzer.analyze(caseCall); + + assertTrue(result.isPresent()); + RangeAggregationBuilder builder = result.get(); + + String expectedJson = + """ + { + "age_groups" : { + "range" : { + "field" : "age", + "ranges" : [ + { + "key" : "adult", + "from" : 18.0 + }, + { + "key" : "minor", + "to" : 18.0 + } + ], + "keyed" : true + } + } + }"""; + + assertEquals(normalizeJson(expectedJson), normalizeJson(builder.toString())); + } + + @Test + void testMultipleConditionsGenerateExpectedDSL() { + // CASE + // WHEN age >= 65 THEN 'senior' + // WHEN age >= 18 THEN 'adult' + // ELSE 'minor' + // END + + RexLiteral literal65 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(65)); + RexLiteral literal18 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(18)); + RexLiteral seniorLiteral = rexBuilder.makeLiteral("senior"); + RexLiteral adultLiteral = rexBuilder.makeLiteral("adult"); + RexLiteral minorLiteral = rexBuilder.makeLiteral("minor"); + + RexCall condition1 = + (RexCall) + rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, fieldRef, literal65); + RexCall condition2 = + (RexCall) + rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, fieldRef, literal18); + + RexCall caseCall = + (RexCall) + rexBuilder.makeCall( + SqlStdOperatorTable.CASE, + Arrays.asList(condition1, seniorLiteral, condition2, adultLiteral, minorLiteral)); + + CaseRangeAnalyzer analyzer = CaseRangeAnalyzer.create("age_categories", rowType); + Optional result = analyzer.analyze(caseCall); + + assertTrue(result.isPresent()); + RangeAggregationBuilder builder = result.get(); + + String expectedJson = + """ + { + "age_categories" : { + "range" : { + "field" : "age", + "ranges" : [ + { + "key" : "senior", + "from" : 65.0 + }, + { + "key" : "adult", + "from" : 18.0, + "to" : 65.0 + }, + { + "key" : "minor", + "to" : 18.0 + } + ], + "keyed" : true + } + } + }"""; + + assertEquals(normalizeJson(expectedJson), normalizeJson(builder.toString())); + } + + @Test + void testLessThanConditionGeneratesExpectedDSL() { + // CASE WHEN age < 21 THEN 'underage' ELSE 'legal' END + + RexLiteral literal21 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(21)); + RexLiteral underageLiteral = rexBuilder.makeLiteral("underage"); + RexLiteral legalLiteral = rexBuilder.makeLiteral("legal"); + + RexCall condition = + (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, fieldRef, literal21); + + RexCall caseCall = + (RexCall) + rexBuilder.makeCall( + SqlStdOperatorTable.CASE, Arrays.asList(condition, underageLiteral, legalLiteral)); + + CaseRangeAnalyzer analyzer = CaseRangeAnalyzer.create("legal_status", rowType); + Optional result = analyzer.analyze(caseCall); + + assertTrue(result.isPresent()); + RangeAggregationBuilder builder = result.get(); + + String expectedJson = + """ + { + "legal_status" : { + "range" : { + "field" : "age", + "ranges" : [ + { + "key" : "underage", + "to" : 21.0 + }, + { + "key" : "legal", + "from" : 21.0 + } + ], + "keyed" : true + } + } + }"""; + + assertEquals(normalizeJson(expectedJson), normalizeJson(builder.toString())); + } + + @Test + void testNullElseClauseGeneratesExpectedDSL() { + // CASE WHEN age >= 18 THEN 'adult' ELSE NULL END + + RexLiteral literal18 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(18)); + RexLiteral adultLiteral = rexBuilder.makeLiteral("adult"); + RexLiteral nullLiteral = + rexBuilder.makeNullLiteral(typeFactory.createSqlType(SqlTypeName.VARCHAR)); + + RexCall condition = + (RexCall) + rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, fieldRef, literal18); + + RexCall caseCall = + (RexCall) + rexBuilder.makeCall( + SqlStdOperatorTable.CASE, Arrays.asList(condition, adultLiteral, nullLiteral)); + + CaseRangeAnalyzer analyzer = CaseRangeAnalyzer.create("adult_check", rowType); + Optional result = analyzer.analyze(caseCall); + + assertTrue(result.isPresent()); + RangeAggregationBuilder builder = result.get(); + + String expectedJson = + """ + { + "adult_check" : { + "range" : { + "field" : "age", + "ranges" : [ + { + "key" : "adult", + "from" : 18.0 + }, + { + "key" : "null", + "to" : 18.0 + } + ], + "keyed" : true + } + } + }"""; + + assertEquals(normalizeJson(expectedJson), normalizeJson(builder.toString())); + } + + @Test + void testSearchConditionGeneratesExpectedDSL() { + // Create a SEARCH condition (Sarg-based range): 18 <= age < 65 + TreeRangeSet rangeSet = TreeRangeSet.create(); + rangeSet.add(Range.closedOpen(BigDecimal.valueOf(18), BigDecimal.valueOf(65))); + + Sarg sarg = Sarg.of(RexUnknownAs.UNKNOWN, rangeSet); + RexNode sargLiteral = + rexBuilder.makeSearchArgumentLiteral(sarg, typeFactory.createSqlType(SqlTypeName.DECIMAL)); + + RexCall searchCall = + (RexCall) + rexBuilder.makeCall(SqlStdOperatorTable.SEARCH, Arrays.asList(fieldRef, sargLiteral)); + + RexLiteral workingLiteral = rexBuilder.makeLiteral("working_age"); + RexLiteral otherLiteral = rexBuilder.makeLiteral("other"); + + RexCall caseCall = + (RexCall) + rexBuilder.makeCall( + SqlStdOperatorTable.CASE, Arrays.asList(searchCall, workingLiteral, otherLiteral)); + + CaseRangeAnalyzer analyzer = CaseRangeAnalyzer.create("employment_status", rowType); + Optional result = analyzer.analyze(caseCall); + + assertTrue(result.isPresent()); + RangeAggregationBuilder builder = result.get(); + + String expectedJson = + """ + { + "employment_status" : { + "range" : { + "field" : "age", + "ranges" : [ + { + "key" : "working_age", + "from" : 18.0, + "to" : 65.0 + }, + { + "key" : "other", + "to" : 18.0 + }, + { + "key" : "other", + "from" : 65.0 + } + ], + "keyed" : true + } + } + }"""; + + assertEquals(normalizeJson(expectedJson), normalizeJson(builder.toString())); + } + + @Test + void testSearchWithDiscontinuousRanges() { + // age >= 20 && age < 30 -> '20-30' + // age >= 40 && age <50 -> '40-50' + // Create discontinuous ranges: [20, 30) and [40, 50) + TreeRangeSet rangeSet = TreeRangeSet.create(); + rangeSet.add(Range.closedOpen(BigDecimal.valueOf(20), BigDecimal.valueOf(30))); + rangeSet.add(Range.closedOpen(BigDecimal.valueOf(40), BigDecimal.valueOf(50))); + + Sarg sarg = Sarg.of(RexUnknownAs.UNKNOWN, rangeSet); + RexNode sargLiteral = + rexBuilder.makeSearchArgumentLiteral(sarg, typeFactory.createSqlType(SqlTypeName.DECIMAL)); + + RexCall searchCall = + (RexCall) + rexBuilder.makeCall(SqlStdOperatorTable.SEARCH, Arrays.asList(fieldRef, sargLiteral)); + + RexLiteral targetLiteral = rexBuilder.makeLiteral("target_age"); + RexLiteral otherLiteral = + rexBuilder.makeNullLiteral(typeFactory.createSqlType(SqlTypeName.VARCHAR)); + + RexCall caseCall = + (RexCall) + rexBuilder.makeCall( + SqlStdOperatorTable.CASE, Arrays.asList(searchCall, targetLiteral, otherLiteral)); + + CaseRangeAnalyzer analyzer = CaseRangeAnalyzer.create("discontinuous_ranges", rowType); + Optional result = analyzer.analyze(caseCall); + + assertTrue(result.isPresent()); + RangeAggregationBuilder builder = result.get(); + + String expectedJson = + """ + { + "discontinuous_ranges" : { + "range" : { + "field" : "age", + "ranges" : [ + { + "key" : "target_age", + "from" : 20.0, + "to" : 30.0 + }, + { + "key" : "target_age", + "from" : 40.0, + "to" : 50.0 + }, + { + "key" : "null", + "to" : 20.0 + }, + { + "key" : "null", + "from" : 30.0, + "to" : 40.0 + }, + { + "key" : "null", + "from" : 50.0 + } + ], + "keyed" : true + } + } + }"""; + + assertEquals(normalizeJson(expectedJson), normalizeJson(builder.toString())); + } + + /** + * Helper method to normalize JSON strings for comparison by removing extra whitespace and + * ensuring consistent formatting. + */ + private String normalizeJson(String json) { + return json.replaceAll("\\s+", " ").replaceAll("\\s*([{}\\[\\],:]?)\\s*", "$1").trim(); + } + + @Test + void testAnalyzeSearchConditionWithInvalidField() { + // Create a SEARCH condition with non-field reference + TreeRangeSet rangeSet = TreeRangeSet.create(); + rangeSet.add(Range.closedOpen(BigDecimal.valueOf(18), BigDecimal.valueOf(65))); + + Sarg sarg = Sarg.of(RexUnknownAs.UNKNOWN, rangeSet); + RexNode sargLiteral = + rexBuilder.makeSearchArgumentLiteral(sarg, typeFactory.createSqlType(SqlTypeName.DECIMAL)); + RexLiteral constantLiteral = rexBuilder.makeExactLiteral(BigDecimal.valueOf(42)); + + RexCall searchCall = + (RexCall) + rexBuilder.makeCall( + SqlStdOperatorTable.SEARCH, + Arrays.asList(constantLiteral, sargLiteral)); // constant instead of field + + RexLiteral resultLiteral = rexBuilder.makeLiteral("result"); + RexLiteral elseLiteral = rexBuilder.makeLiteral("else"); + + RexCall caseCall = + (RexCall) + rexBuilder.makeCall( + SqlStdOperatorTable.CASE, Arrays.asList(searchCall, resultLiteral, elseLiteral)); + + CaseRangeAnalyzer analyzer = CaseRangeAnalyzer.create("test", rowType); + Optional result = analyzer.analyze(caseCall); + + assertFalse(result.isPresent()); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java index e4c7220a39f..09348cae437 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java @@ -408,7 +408,7 @@ void test_push_down_project() { .size(MAX_RESULT_WINDOW) .timeout(DEFAULT_QUERY_TIMEOUT) .sort(DOC_FIELD_NAME, ASC) - .sort(METADATA_FIELD_ID, ASC) + .sort(SortBuilders.shardDocSort()) .pointInTimeBuilder(new PointInTimeBuilder("samplePITId")) .fetchSource(new String[] {"intA"}, new String[0]), requestBuilder); @@ -421,7 +421,7 @@ void test_push_down_project() { .size(MAX_RESULT_WINDOW) .timeout(DEFAULT_QUERY_TIMEOUT) .sort(DOC_FIELD_NAME, ASC) - .sort(METADATA_FIELD_ID, ASC) + .sort(SortBuilders.shardDocSort()) .pointInTimeBuilder(new PointInTimeBuilder("samplePITId")) .fetchSource("intA", null), exprValueFactory, @@ -536,7 +536,7 @@ void test_push_down_project_with_alias_type() { .size(MAX_RESULT_WINDOW) .timeout(DEFAULT_QUERY_TIMEOUT) .sort(DOC_FIELD_NAME, ASC) - .sort(METADATA_FIELD_ID, ASC) + .sort(SortBuilders.shardDocSort()) .pointInTimeBuilder(new PointInTimeBuilder("samplePITId")) .fetchSource(new String[] {"intA"}, new String[0]), requestBuilder); @@ -549,7 +549,7 @@ void test_push_down_project_with_alias_type() { .size(MAX_RESULT_WINDOW) .timeout(DEFAULT_QUERY_TIMEOUT) .sort(DOC_FIELD_NAME, ASC) - .sort(METADATA_FIELD_ID, ASC) + .sort(SortBuilders.shardDocSort()) .pointInTimeBuilder(new PointInTimeBuilder("samplePITId")) .fetchSource("intA", null), exprValueFactory, diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java index 0ed865705a7..9d7a6b93f92 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java @@ -18,8 +18,6 @@ import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.volcano.VolcanoPlanner; import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.rel.type.RelDataTypeSystem; import org.apache.calcite.rel.type.StructKind; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexInputRef; @@ -27,7 +25,6 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.runtime.Hook; import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.sql.type.SqlTypeFactoryImpl; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.Holder; import org.junit.jupiter.api.Test; @@ -47,6 +44,8 @@ import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.index.query.TermsQueryBuilder; import org.opensearch.index.query.WildcardQueryBuilder; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.function.BuiltinFunctionName; @@ -57,23 +56,28 @@ import org.opensearch.sql.opensearch.request.PredicateAnalyzer.QueryExpression; public class PredicateAnalyzerTest { - final RelDataTypeFactory typeFactory = new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT); + final OpenSearchTypeFactory typeFactory = OpenSearchTypeFactory.TYPE_FACTORY; final RexBuilder builder = new RexBuilder(typeFactory); final RelOptCluster cluster = RelOptCluster.create(new VolcanoPlanner(), builder); - final List schema = List.of("a", "b", "c"); + final List schema = List.of("a", "b", "c", "d"); final Map fieldTypes = Map.of( "a", OpenSearchDataType.of(MappingType.Integer), "b", OpenSearchDataType.of( MappingType.Text, Map.of("fields", Map.of("keyword", Map.of("type", "keyword")))), - "c", OpenSearchDataType.of(MappingType.Text)); // Text without keyword cannot be push down + "c", OpenSearchDataType.of(MappingType.Text), // Text without keyword cannot be push down + "d", OpenSearchDataType.of(MappingType.Date)); final RexInputRef field1 = builder.makeInputRef(typeFactory.createSqlType(SqlTypeName.INTEGER), 0); final RexInputRef field2 = builder.makeInputRef(typeFactory.createSqlType(SqlTypeName.VARCHAR), 1); + final RexInputRef field4 = builder.makeInputRef(typeFactory.createUDT(ExprUDT.EXPR_TIMESTAMP), 3); final RexLiteral numericLiteral = builder.makeExactLiteral(new BigDecimal(12)); final RexLiteral stringLiteral = builder.makeLiteral("Hi"); + final RexNode dateTimeLiteral = + builder.makeLiteral( + "1987-02-03 04:34:56", typeFactory.createUDT(ExprUDT.EXPR_TIMESTAMP), true); final RexNode aliasedField2 = builder.makeCall( SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, builder.makeLiteral("field"), field2); @@ -612,6 +616,7 @@ void likeFunction_textField_scriptPushDown() throws ExpressionNotAnalyzableExcep .kind(StructKind.FULLY_QUALIFIED) .add("a", builder.getTypeFactory().createSqlType(SqlTypeName.BIGINT)) .add("b", builder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR)) + .add("c", builder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR)) .build(); Hook.CURRENT_TIME.addThread((Consumer>) h -> h.set(0L)); QueryExpression expression = @@ -739,11 +744,11 @@ void equals_scriptPushDown_Struct() throws ExpressionNotAnalyzableException { .kind(StructKind.FULLY_QUALIFIED) .add("d", mapType) .build(); - final RexInputRef field4 = builder.makeInputRef(mapType, 3); + final RexInputRef field4 = builder.makeInputRef(mapType, 0); final Map newFieldTypes = Map.of("d", OpenSearchDataType.of(ExprCoreType.STRUCT)); final List newSchema = List.of("d"); - RexNode call = builder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, field4); + RexNode call = builder.makeCall(SqlStdOperatorTable.IS_EMPTY, field4); Hook.CURRENT_TIME.addThread((Consumer>) h -> h.set(0L)); QueryBuilder builder = PredicateAnalyzer.analyze(call, newSchema, newFieldTypes, rowType, cluster); @@ -974,4 +979,94 @@ void queryStringWithoutFields_generatesQueryStringQuery() }""", result.toString()); } + + @Test + void equals_generatesRangeQueryForDateTime() throws ExpressionNotAnalyzableException { + RexNode call = builder.makeCall(SqlStdOperatorTable.EQUALS, field4, dateTimeLiteral); + QueryBuilder result = PredicateAnalyzer.analyze(call, schema, fieldTypes); + + assertInstanceOf(RangeQueryBuilder.class, result); + assertEquals( + """ + { + "range" : { + "d" : { + "from" : "1987-02-03T04:34:56.000Z", + "to" : "1987-02-03T04:34:56.000Z", + "include_lower" : true, + "include_upper" : true, + "format" : "date_time", + "boost" : 1.0 + } + } + }""", + result.toString()); + } + + @Test + void notEquals_generatesBoolQueryForDateTime() throws ExpressionNotAnalyzableException { + RexNode call = builder.makeCall(SqlStdOperatorTable.NOT_EQUALS, field4, dateTimeLiteral); + QueryBuilder result = PredicateAnalyzer.analyze(call, schema, fieldTypes); + + assertInstanceOf(BoolQueryBuilder.class, result); + assertEquals( + """ + { + "bool" : { + "should" : [ + { + "range" : { + "d" : { + "from" : "1987-02-03T04:34:56.000Z", + "to" : null, + "include_lower" : false, + "include_upper" : true, + "format" : "date_time", + "boost" : 1.0 + } + } + }, + { + "range" : { + "d" : { + "from" : null, + "to" : "1987-02-03T04:34:56.000Z", + "include_lower" : true, + "include_upper" : false, + "format" : "date_time", + "boost" : 1.0 + } + } + } + ], + "adjust_pure_negative" : true, + "boost" : 1.0 + } + }""", + result.toString()); + } + + @Test + void gte_generatesRangeQueryWithFormatForDateTime() throws ExpressionNotAnalyzableException { + RexNode call = + builder.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, field4, dateTimeLiteral); + QueryBuilder result = PredicateAnalyzer.analyze(call, schema, fieldTypes); + + assertInstanceOf(RangeQueryBuilder.class, result); + assertEquals( + """ + { + "range" : { + "d" : { + "from" : "1987-02-03T04:34:56.000Z", + "to" : null, + "include_lower" : true, + "include_upper" : true, + "format" : "date_time", + "boost" : 1.0 + } + } + }""", + result.toString()); + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java index 57f7c4ea044..5dc88ad5d64 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java @@ -276,16 +276,16 @@ void top_hits_aggregation_should_pass() { + " \"_index\": \"accounts\",\n" + " \"_id\": \"1\",\n" + " \"_score\": 1.0,\n" - + " \"_source\": {\n" - + " \"gender\": \"m\"\n" + + " \"fields\": {\n" + + " \"gender\": [\"m\"]\n" + " }\n" + " },\n" + " {\n" + " \"_index\": \"accounts\",\n" + " \"_id\": \"2\",\n" + " \"_score\": 1.0,\n" - + " \"_source\": {\n" - + " \"gender\": \"f\"\n" + + " \"fields\": {\n" + + " \"gender\": [\"f\"]\n" + " }\n" + " }\n" + " ]\n" diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/CalciteIndexScanCostTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/CalciteIndexScanCostTest.java index f02b40b9eae..c67d7cfaa3e 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/CalciteIndexScanCostTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/CalciteIndexScanCostTest.java @@ -49,6 +49,13 @@ import org.opensearch.sql.common.setting.Settings.Key; import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder; import org.opensearch.sql.opensearch.storage.OpenSearchIndex; +import org.opensearch.sql.opensearch.storage.scan.context.AggPushDownAction; +import org.opensearch.sql.opensearch.storage.scan.context.AggregationBuilderAction; +import org.opensearch.sql.opensearch.storage.scan.context.FilterDigest; +import org.opensearch.sql.opensearch.storage.scan.context.LimitDigest; +import org.opensearch.sql.opensearch.storage.scan.context.OSRequestBuilderAction; +import org.opensearch.sql.opensearch.storage.scan.context.PushDownOperation; +import org.opensearch.sql.opensearch.storage.scan.context.PushDownType; @ExtendWith(MockitoExtension.class) public class CalciteIndexScanCostTest { diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanOptimizationTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanOptimizationTest.java index 06cc0b82fd7..ca5cc712ae9 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanOptimizationTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanOptimizationTest.java @@ -74,7 +74,7 @@ import org.opensearch.sql.expression.function.OpenSearchFunctions; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder; -import org.opensearch.sql.opensearch.response.agg.CompositeAggregationParser; +import org.opensearch.sql.opensearch.response.agg.BucketAggregationParser; import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; import org.opensearch.sql.opensearch.response.agg.SingleValueParser; import org.opensearch.sql.opensearch.storage.script.aggregation.AggregationQueryBuilder; @@ -802,8 +802,7 @@ private Runnable withAggregationPushedDown( AggregationBuilders.avg(aggregation.aggregateName).field(aggregation.aggregateBy)) .size(AggregationQueryBuilder.AGGREGATION_BUCKET_SIZE); List aggBuilders = Collections.singletonList(aggBuilder); - responseParser = - new CompositeAggregationParser(new SingleValueParser(aggregation.aggregateName)); + responseParser = new BucketAggregationParser(new SingleValueParser(aggregation.aggregateName)); return () -> { verify(requestBuilder, times(1)).pushDownAggregation(Pair.of(aggBuilders, responseParser)); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java index 0a15df95b29..05d0c78ce31 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java @@ -51,6 +51,7 @@ public class OpenSearchIndexScanPaginationTest { @BeforeEach void setup() { lenient().when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); + lenient().when(settings.getSettingValue(Settings.Key.QUERY_BUCKET_SIZE)).thenReturn(1000); lenient() .when(settings.getSettingValue(Settings.Key.SQL_CURSOR_KEEP_ALIVE)) .thenReturn(TimeValue.timeValueMinutes(1)); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java index 64ae7b187c2..eeacbecd7aa 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java @@ -409,10 +409,9 @@ void should_build_top_hits_aggregation() { + " \"version\" : false,%n" + " \"seq_no_primary_term\" : false,%n" + " \"explain\" : false,%n" - + " \"_source\" : {%n" - + " \"includes\" : [ \"name\" ],%n" - + " \"excludes\" : [ ]%n" - + " }%n" + + " \"fields\" : [ {%n" + + " \"field\" : \"name\"%n" + + " } ]%n" + " }%n" + " }%n" + "}"), @@ -449,10 +448,9 @@ void should_build_filtered_top_hits_aggregation() { + " \"version\" : false,%n" + " \"seq_no_primary_term\" : false,%n" + " \"explain\" : false,%n" - + " \"_source\" : {%n" - + " \"includes\" : [ \"name\" ],%n" - + " \"excludes\" : [ ]%n" - + " }%n" + + " \"fields\" : [ {%n" + + " \"field\" : \"name\"%n" + + " } ]%n" + " }%n" + " }%n" + " }%n" diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializerTest.java index eee479f9504..1ba4d94c614 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializerTest.java @@ -125,4 +125,35 @@ void testDeserializeFunctionOutOfScope() { String code = serializer.serialize(outOfScopeRex, rowType, fieldTypes); assertThrows(IllegalStateException.class, () -> serializer.deserialize(code)); } + + @Test + void testSerializeIndexRemappedRexNode() { + RelDataType originalRowType = + rexBuilder + .getTypeFactory() + .builder() + .kind(StructKind.FULLY_QUALIFIED) + .add("Firstname", rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR)) + .add("Referer", rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR)) + .build(); + Map originalFieldTypes = + Map.of("Referer", ExprCoreType.STRING, "Firstname", ExprCoreType.STRING); + RexNode originalRexUpper = + PPLFuncImpTable.INSTANCE.resolve( + rexBuilder, + BuiltinFunctionName.UPPER, + rexBuilder.makeInputRef(originalRowType.getFieldList().get(1).getType(), 1)); + RexNode remappedRexUpper = + PPLFuncImpTable.INSTANCE.resolve( + rexBuilder, + BuiltinFunctionName.UPPER, + rexBuilder.makeInputRef(rowType.getFieldList().get(0).getType(), 0)); + + String code = serializer.serialize(originalRexUpper, originalRowType, originalFieldTypes); + Map objects = serializer.deserialize(code); + + assertEquals(remappedRexUpper, objects.get(RelJsonSerializer.EXPR)); + assertEquals(rowType, objects.get(RelJsonSerializer.ROW_TYPE)); + assertEquals(fieldTypes, objects.get(RelJsonSerializer.FIELD_TYPES)); + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtilTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtilTest.java index 60cfba5ba7e..a9790d6485e 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtilTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtilTest.java @@ -9,6 +9,8 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.Arrays; +import java.util.List; import java.util.Optional; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; @@ -243,4 +245,84 @@ private void assertExpectedInputInfo( assertEquals(index, result.get().getLeft().intValue()); assertEquals(flipped, result.get().getRight()); } + + @Test + public void testScenario1() { + List input = Arrays.asList("a_b", "a.b"); + List expected = Arrays.asList("a_b", "a_b0"); + List result = OpenSearchRelOptUtil.resolveColumnNameConflicts(input); + assertEquals(expected, result); + } + + @Test + public void testScenario2() { + List input = Arrays.asList("a_b", "a_b0", "a.b"); + List expected = Arrays.asList("a_b", "a_b0", "a_b1"); + List result = OpenSearchRelOptUtil.resolveColumnNameConflicts(input); + assertEquals(expected, result); + } + + @Test + public void testScenario3() { + List input = Arrays.asList("a_b", "a_b1", "a.b"); + List expected = Arrays.asList("a_b", "a_b1", "a_b0"); + List result = OpenSearchRelOptUtil.resolveColumnNameConflicts(input); + assertEquals(expected, result); + } + + @Test + public void testScenario4() { + List input = Arrays.asList("a_b0", "a.b0", "a.b1"); + List expected = Arrays.asList("a_b0", "a_b00", "a_b1"); + List result = OpenSearchRelOptUtil.resolveColumnNameConflicts(input); + assertEquals(expected, result); + } + + @Test + public void testMultipleDots() { + List input = Arrays.asList("a.b.c", "a_b_c", "a.b.c"); + List expected = Arrays.asList("a_b_c0", "a_b_c", "a_b_c1"); + List result = OpenSearchRelOptUtil.resolveColumnNameConflicts(input); + assertEquals(expected, result); + } + + @Test + public void testComplexScenario() { + List input = Arrays.asList("x", "x", "x", "x"); + List expected = Arrays.asList("x", "x", "x", "x"); + List result = OpenSearchRelOptUtil.resolveColumnNameConflicts(input); + assertEquals(expected, result); + } + + @Test + public void testNoConflict() { + List input = Arrays.asList("col1", "col2", "col3"); + List expected = Arrays.asList("col1", "col2", "col3"); + List result = OpenSearchRelOptUtil.resolveColumnNameConflicts(input); + assertEquals(expected, result); + } + + @Test + public void testMixedConflict() { + List input = Arrays.asList("a.b", "a_b", "a.b", "a_b0"); + List expected = Arrays.asList("a_b1", "a_b", "a_b2", "a_b0"); + List result = OpenSearchRelOptUtil.resolveColumnNameConflicts(input); + assertEquals(expected, result); + } + + @Test + public void testOriginalNamesPreserved() { + List input = Arrays.asList("endpoint.ip", "account.id", "timestamp"); + List expected = Arrays.asList("endpoint_ip", "account_id", "timestamp"); + List result = OpenSearchRelOptUtil.resolveColumnNameConflicts(input); + assertEquals(expected, result); + } + + @Test + public void testNoDots() { + List input = Arrays.asList("col1", "col2", "col3"); + List expected = Arrays.asList("col1", "col2", "col3"); + List result = OpenSearchRelOptUtil.resolveColumnNameConflicts(input); + assertEquals(expected, result); + } } diff --git a/plugin/build.gradle b/plugin/build.gradle index c6d05e934fa..fd30ccdf931 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -44,8 +44,12 @@ ext { } repositories { + mavenLocal() mavenCentral() - maven { url 'https://jitpack.io' } + maven { + url 'https://jitpack.io' + content { includeGroup "com.github.babbel" } + } } opensearchplugin { @@ -111,7 +115,7 @@ configurations.all { resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" resolutionStrategy.force "com.squareup.okhttp3:okhttp:4.12.0" resolutionStrategy.force "joda-time:joda-time:2.10.12" - resolutionStrategy.force "org.slf4j:slf4j-api:1.7.36" + resolutionStrategy.force "org.slf4j:slf4j-api:2.0.13" resolutionStrategy.force "org.apache.httpcomponents:httpcore:4.4.15" resolutionStrategy.force "org.apache.httpcomponents:httpclient:4.5.13" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10" diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index fe4be96fc07..efd7a39d3c5 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -26,6 +26,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Guice; import org.opensearch.common.inject.Injector; import org.opensearch.common.inject.ModulesBuilder; import org.opensearch.common.settings.ClusterSettings; @@ -259,7 +260,7 @@ public Collection createComponents( }); modules.add(new AsyncExecutorServiceModule()); modules.add(new DirectQueryModule()); - injector = modules.createInjector(); + injector = Guice.createInjector(modules); ClusterManagerEventListener clusterManagerEventListener = new ClusterManagerEventListener( clusterService, diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java index 94cc8c2fe0f..81b2b5d26dc 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java @@ -93,7 +93,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient nod new ActionListener<>() { @Override public void onResponse(TransportPPLQueryResponse response) { - sendResponse(channel, OK, response.getResult()); + sendResponse(channel, OK, response.getContentType(), response.getResult()); } @Override @@ -129,8 +129,9 @@ public void onFailure(Exception e) { }); } - private void sendResponse(RestChannel channel, RestStatus status, String content) { - channel.sendResponse(new BytesRestResponse(status, "application/json; charset=UTF-8", content)); + private void sendResponse( + RestChannel channel, RestStatus status, String contentType, String content) { + channel.sendResponse(new BytesRestResponse(status, contentType, content)); } private void reportError(final RestChannel channel, final Exception e, final RestStatus status) { diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java index fc257931c2a..b53e41b0ee4 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java @@ -6,6 +6,7 @@ package org.opensearch.sql.plugin.transport; import static org.opensearch.rest.BaseRestHandler.MULTI_ALLOW_EXPLICIT_INDEX; +import static org.opensearch.sql.executor.ExecutionEngine.ExplainResponse.normalizeLf; import static org.opensearch.sql.lang.PPLLangSpec.PPL_SPEC; import static org.opensearch.sql.protocol.response.format.JsonResponseFormatter.Style.PRETTY; @@ -16,6 +17,7 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Guice; import org.opensearch.common.inject.Inject; import org.opensearch.common.inject.Injector; import org.opensearch.common.inject.ModulesBuilder; @@ -41,6 +43,7 @@ import org.opensearch.sql.protocol.response.format.ResponseFormatter; import org.opensearch.sql.protocol.response.format.SimpleJsonResponseFormatter; import org.opensearch.sql.protocol.response.format.VisualizationResponseFormatter; +import org.opensearch.sql.protocol.response.format.YamlResponseFormatter; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; import org.opensearch.transport.client.node.NodeClient; @@ -73,7 +76,7 @@ public TransportPPLQueryAction( .toInstance(new OpenSearchSettings(clusterService.getClusterSettings())); b.bind(DataSourceService.class).toInstance(dataSourceService); }); - this.injector = modules.createInjector(); + this.injector = Guice.createInjector(modules); this.pplEnabled = () -> MULTI_ALLOW_EXPLICIT_INDEX.get(clusterSettings) @@ -109,12 +112,13 @@ protected void doExecute( PPLQueryRequest transformedRequest = transportRequest.toPPLQueryRequest(); if (transformedRequest.isExplainRequest()) { - pplService.explain(transformedRequest, createExplainResponseListener(listener)); + pplService.explain( + transformedRequest, createExplainResponseListener(transformedRequest, listener)); } else { pplService.execute( transformedRequest, createListener(transformedRequest, listener), - createExplainResponseListener(listener)); + createExplainResponseListener(transformedRequest, listener)); } } @@ -124,18 +128,32 @@ protected void doExecute( * legacy module. */ private ResponseListener createExplainResponseListener( - ActionListener listener) { + PPLQueryRequest request, ActionListener listener) { return new ResponseListener() { @Override public void onResponse(ExecutionEngine.ExplainResponse response) { - String responseContent = - new JsonResponseFormatter(PRETTY) { - @Override - protected Object buildJsonObject(ExecutionEngine.ExplainResponse response) { - return response; - } - }.format(response); - listener.onResponse(new TransportPPLQueryResponse(responseContent)); + Optional isYamlFormat = + Format.ofExplain(request.getFormat()).filter(format -> format.equals(Format.YAML)); + ResponseFormatter formatter; + if (isYamlFormat.isPresent()) { + formatter = + new YamlResponseFormatter<>() { + @Override + protected Object buildYamlObject(ExecutionEngine.ExplainResponse response) { + return normalizeLf(response); + } + }; + } else { + formatter = + new JsonResponseFormatter<>(PRETTY) { + @Override + protected Object buildJsonObject(ExecutionEngine.ExplainResponse response) { + return response; + } + }; + } + listener.onResponse( + new TransportPPLQueryResponse(formatter.format(response), formatter.contentType())); } @Override diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryResponse.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryResponse.java index a8d06fa6264..8411b974a2f 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryResponse.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryResponse.java @@ -10,25 +10,36 @@ import java.io.IOException; import java.io.UncheckedIOException; import lombok.Getter; -import lombok.RequiredArgsConstructor; import org.opensearch.core.action.ActionResponse; import org.opensearch.core.common.io.stream.InputStreamStreamInput; import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; -@RequiredArgsConstructor public class TransportPPLQueryResponse extends ActionResponse { @Getter private final String result; + @Getter private final String contentType; + + public TransportPPLQueryResponse(String result) { + this.result = result; + this.contentType = "application/json; charset=UTF-8"; + } + + public TransportPPLQueryResponse(String result, String contentType) { + this.result = result; + this.contentType = contentType; + } public TransportPPLQueryResponse(StreamInput in) throws IOException { super(in); result = in.readString(); + contentType = in.readString(); } @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(result); + out.writeString(contentType); } public static TransportPPLQueryResponse fromActionResponse(ActionResponse actionResponse) { diff --git a/ppl/build.gradle b/ppl/build.gradle index 8d00d7984ca..3e244c6fa01 100644 --- a/ppl/build.gradle +++ b/ppl/build.gradle @@ -63,7 +63,7 @@ dependencies { testImplementation group: 'junit', name: 'junit', version: '4.13.2' testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: "${hamcrest_version}" testImplementation group: 'org.mockito', name: 'mockito-core', version: "${mockito_version}" - testImplementation group: 'org.apache.calcite', name: 'calcite-testkit', version: '1.38.0' + testImplementation group: 'org.apache.calcite', name: 'calcite-testkit', version: '1.41.0' testImplementation(testFixtures(project(":core"))) } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 96ca6c4d835..511122fa28c 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -22,6 +22,7 @@ TABLE: 'TABLE'; // Alias for FIELDS command RENAME: 'RENAME'; STATS: 'STATS'; EVENTSTATS: 'EVENTSTATS'; +STREAMSTATS: 'STREAMSTATS'; DEDUP: 'DEDUP'; SORT: 'SORT'; EVAL: 'EVAL'; @@ -85,6 +86,7 @@ DESC: 'DESC'; DATASOURCES: 'DATASOURCES'; USING: 'USING'; WITH: 'WITH'; +VALUE: 'VALUE'; SIMPLE: 'SIMPLE'; STANDARD: 'STANDARD'; COST: 'COST'; @@ -109,7 +111,13 @@ DEDUP_SPLITVALUES: 'DEDUP_SPLITVALUES'; PARTITIONS: 'PARTITIONS'; ALLNUM: 'ALLNUM'; DELIM: 'DELIM'; +CURRENT: 'CURRENT'; +WINDOW: 'WINDOW'; +GLOBAL: 'GLOBAL'; +RESET_BEFORE: 'RESET_BEFORE'; +RESET_AFTER: 'RESET_AFTER'; BUCKET_NULLABLE: 'BUCKET_NULLABLE'; +USENULL: 'USENULL'; CENTROIDS: 'CENTROIDS'; ITERATIONS: 'ITERATIONS'; DISTANCE_TYPE: 'DISTANCE_TYPE'; @@ -125,6 +133,7 @@ TIME_ZONE: 'TIME_ZONE'; TRAINING_DATA_SIZE: 'TRAINING_DATA_SIZE'; ANOMALY_SCORE_THRESHOLD: 'ANOMALY_SCORE_THRESHOLD'; APPEND: 'APPEND'; +MULTISEARCH: 'MULTISEARCH'; COUNTFIELD: 'COUNTFIELD'; SHOWCOUNT: 'SHOWCOUNT'; LIMIT: 'LIMIT'; @@ -167,8 +176,8 @@ HOUR_MINUTE: 'HOUR_MINUTE'; HOUR_OF_DAY: 'HOUR_OF_DAY'; HOUR_SECOND: 'HOUR_SECOND'; INTERVAL: 'INTERVAL'; -MICROSECOND: 'MICROSECOND'; MILLISECOND: 'MILLISECOND'; +MICROSECOND: 'MICROSECOND'; MINUTE: 'MINUTE'; MINUTE_MICROSECOND: 'MINUTE_MICROSECOND'; MINUTE_OF_DAY: 'MINUTE_OF_DAY'; @@ -186,9 +195,7 @@ YEAR: 'YEAR'; YEAR_MONTH: 'YEAR_MONTH'; // DATASET TYPES -DATAMODEL: 'DATAMODEL'; LOOKUP: 'LOOKUP'; -SAVEDSEARCH: 'SAVEDSEARCH'; // CONVERTED DATA TYPES INT: 'INT'; @@ -396,7 +403,6 @@ SUBSTRING: 'SUBSTRING'; LTRIM: 'LTRIM'; RTRIM: 'RTRIM'; TRIM: 'TRIM'; -TO: 'TO'; LOWER: 'LOWER'; UPPER: 'UPPER'; CONCAT: 'CONCAT'; @@ -424,6 +430,7 @@ ISBLANK: 'ISBLANK'; // COLLECTION FUNCTIONS ARRAY: 'ARRAY'; ARRAY_LENGTH: 'ARRAY_LENGTH'; +MVAPPEND: 'MVAPPEND'; MVJOIN: 'MVJOIN'; FORALL: 'FORALL'; FILTER: 'FILTER'; @@ -503,7 +510,8 @@ ALIGNTIME: 'ALIGNTIME'; PERCENTILE_SHORTCUT: PERC(INTEGER_LITERAL | DECIMAL_LITERAL) | 'P'(INTEGER_LITERAL | DECIMAL_LITERAL); SPANLENGTH: [0-9]+ ( - 'US'|'MS'|'CS'|'DS' + 'US' |'CS'|'DS' + |'MS'|'MILLISECOND'|'MILLISECONDS' |'S'|'SEC'|'SECS'|'SECOND'|'SECONDS' |'MIN'|'MINS'|'MINUTE'|'MINUTES' |'H'|'HR'|'HRS'|'HOUR'|'HOURS' diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 3ee0c568b41..6b98fac02d6 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -20,7 +20,7 @@ pplStatement ; queryStatement - : pplCommands (PIPE commands)* + : (PIPE)? pplCommands (PIPE commands)* ; explainStatement @@ -43,6 +43,7 @@ pplCommands : describeCommand | showDataSourcesCommand | searchCommand + | multisearchCommand ; commands @@ -53,13 +54,13 @@ commands | renameCommand | statsCommand | eventstatsCommand + | streamstatsCommand | dedupCommand | sortCommand | evalCommand | headCommand | binCommand - | topCommand - | rareCommand + | rareTopCommand | grokCommand | parseCommand | spathCommand @@ -78,6 +79,7 @@ commands | regexCommand | timechartCommand | rexCommand + | replaceCommand ; commandName @@ -91,6 +93,7 @@ commandName | RENAME | STATS | EVENTSTATS + | STREAMSTATS | DEDUP | SORT | EVAL @@ -114,7 +117,9 @@ commandName | REVERSE | REGEX | APPEND + | MULTISEARCH | REX + | REPLACE ; searchCommand @@ -202,6 +207,14 @@ renameCommand : RENAME renameClasue (COMMA? renameClasue)* ; +replaceCommand + : REPLACE replacePair (COMMA replacePair)* IN fieldList + ; + +replacePair + : pattern=stringLiteral WITH replacement=stringLiteral + ; + statsCommand : STATS statsArgs statsAggTerm (COMMA statsAggTerm)* (statsByClause)? (dedupSplitArg)? ; @@ -234,12 +247,40 @@ eventstatsCommand : EVENTSTATS eventstatsAggTerm (COMMA eventstatsAggTerm)* (statsByClause)? ; +streamstatsCommand + : STREAMSTATS streamstatsArgs streamstatsAggTerm (COMMA streamstatsAggTerm)* (statsByClause)? + ; + +streamstatsArgs + : (currentArg | windowArg | globalArg | resetBeforeArg | resetAfterArg)* + ; + +currentArg + : CURRENT EQUAL current = booleanLiteral + ; + +windowArg + : WINDOW EQUAL window = integerLiteral + ; + +globalArg + : GLOBAL EQUAL global = booleanLiteral + ; + +resetBeforeArg + : RESET_BEFORE EQUAL logicalExpression + ; + +resetAfterArg + : RESET_AFTER EQUAL logicalExpression + ; + dedupCommand : DEDUP (number = integerLiteral)? fieldList (KEEPEMPTY EQUAL keepempty = booleanLiteral)? (CONSECUTIVE EQUAL consecutive = booleanLiteral)? ; sortCommand - : SORT (count = integerLiteral)? sortbyClause (ASC | A | DESC | D)? + : SORT (count = integerLiteral)? sortbyClause ; reverseCommand @@ -251,12 +292,8 @@ timechartCommand ; timechartParameter - : (spanClause | SPAN EQUAL spanLiteral) - | timechartArg - ; - -timechartArg : LIMIT EQUAL integerLiteral + | SPAN EQUAL spanLiteral | USEOTHER EQUAL (booleanLiteral | ident) ; @@ -301,12 +338,14 @@ logSpanValue : LOG_WITH_BASE # logWithBaseSpan ; -topCommand - : TOP (number = integerLiteral)? (COUNTFIELD EQUAL countfield = stringLiteral)? (SHOWCOUNT EQUAL showcount = booleanLiteral)? fieldList (byClause)? +rareTopCommand + : (TOP | RARE) (number = integerLiteral)? rareTopOption* fieldList (byClause)? ; -rareCommand - : RARE (number = integerLiteral)? (COUNTFIELD EQUAL countfield = stringLiteral)? (SHOWCOUNT EQUAL showcount = booleanLiteral)? fieldList (byClause)? +rareTopOption + : COUNTFIELD EQUAL countField = stringLiteral + | SHOWCOUNT EQUAL showCount = booleanLiteral + | USENULL EQUAL useNull = booleanLiteral ; grokCommand @@ -415,8 +454,10 @@ lookupPair ; fillnullCommand - : FILLNULL fillNullWith - | FILLNULL fillNullUsing + : FILLNULL fillNullWith # fillNullWithClause + | FILLNULL fillNullUsing # fillNullUsingClause + | FILLNULL VALUE EQUAL replacement = valueExpression fieldList # fillNullValueWithFields + | FILLNULL VALUE EQUAL replacement = valueExpression # fillNullValueAllFields ; fillNullWith @@ -460,6 +501,10 @@ appendCommand : APPEND LT_SQR_PRTHS searchCommand? (PIPE commands)* RT_SQR_PRTHS ; +multisearchCommand + : MULTISEARCH (LT_SQR_PRTHS subSearch RT_SQR_PRTHS)+ + ; + kmeansCommand : KMEANS (kmeansParameter)* ; @@ -517,21 +562,13 @@ tableSourceClause ; dynamicSourceClause - : LT_SQR_PRTHS sourceReferences (COMMA sourceFilterArgs)? RT_SQR_PRTHS - ; - -sourceReferences - : sourceReference (COMMA sourceReference)* + : LT_SQR_PRTHS (sourceReference | sourceFilterArg) (COMMA (sourceReference | sourceFilterArg))* RT_SQR_PRTHS ; sourceReference : (CLUSTER)? wcQualifiedName ; -sourceFilterArgs - : sourceFilterArg (COMMA sourceFilterArg)* - ; - sourceFilterArg : ident EQUAL literalValue | ident IN valueList @@ -607,7 +644,7 @@ bySpanClause ; spanClause - : SPAN LT_PRTHS fieldExpression COMMA value = spanLiteral RT_PRTHS + : SPAN LT_PRTHS (fieldExpression COMMA)? value = spanLiteral RT_PRTHS ; sortbyClause @@ -622,6 +659,10 @@ eventstatsAggTerm : windowFunction (AS alias = wcFieldExpression)? ; +streamstatsAggTerm + : windowFunction (AS alias = wcFieldExpression)? + ; + windowFunction : windowFunctionName LT_PRTHS functionArgs RT_PRTHS ; @@ -659,6 +700,7 @@ statsFunction | takeAggFunction # takeAggFunctionCall | valuesAggFunction # valuesAggFunctionCall | percentileApproxFunction # percentileApproxFunctionCall + | perFunction # perFunctionCall | statsFunctionName LT_PRTHS functionArgs RT_PRTHS # statsFunctionCall ; @@ -695,6 +737,10 @@ percentileApproxFunction COMMA percent = numericLiteral (COMMA compression = numericLiteral)? RT_PRTHS ; +perFunction + : funcName=(PER_SECOND | PER_MINUTE | PER_HOUR | PER_DAY) LT_PRTHS functionArg RT_PRTHS + ; + numericLiteral : integerLiteral | decimalLiteral @@ -800,7 +846,10 @@ fieldList ; sortField - : (PLUS | MINUS)? sortFieldExpression + : (PLUS | MINUS) sortFieldExpression (ASC | A | DESC | D) # invalidMixedSortField + | (PLUS | MINUS) sortFieldExpression # prefixSortField + | sortFieldExpression (ASC | A | DESC | D) # suffixSortField + | sortFieldExpression # defaultSortField ; sortFieldExpression @@ -1005,6 +1054,7 @@ geoipFunctionName collectionFunctionName : ARRAY | ARRAY_LENGTH + | MVAPPEND | MVJOIN | FORALL | EXISTS @@ -1134,6 +1184,7 @@ extractFunctionCall simpleDateTimePart : MICROSECOND + | MILLISECOND | SECOND | MINUTE | HOUR @@ -1312,6 +1363,7 @@ timestampLiteral intervalUnit : MICROSECOND + | MILLISECOND | SECOND | MINUTE | HOUR @@ -1418,6 +1470,7 @@ searchableKeyWord | REGEX | PUNCT | USING + | VALUE | CAST | GET_FORMAT | EXTRACT @@ -1437,7 +1490,13 @@ searchableKeyWord | PARTITIONS | ALLNUM | DELIM + | CURRENT + | WINDOW + | GLOBAL + | RESET_BEFORE + | RESET_AFTER | BUCKET_NULLABLE + | USENULL | CENTROIDS | ITERATIONS | DISTANCE_TYPE @@ -1457,7 +1516,16 @@ searchableKeyWord | PATH | INPUT | OUTPUT - + | AS + | ON + | LIMIT + | OVERWRITE + | FIELD + | SED + | MAX_MATCH + | OFFSET_FIELD + | patternMethod + | patternMode // AGGREGATIONS AND WINDOW | statsFunctionName | windowFunctionName @@ -1497,4 +1565,4 @@ searchableKeyWord | LEFT_HINT | RIGHT_HINT | PERCENTILE_SHORTCUT - ; \ No newline at end of file + ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java b/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java index ccd4d49dd4a..4cb27a32ad0 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java @@ -89,6 +89,7 @@ private AbstractPlan plan( PPLQueryRequest request, ResponseListener queryListener, ResponseListener explainListener) { + log.info("ORIGINAL PPL {}", request.getRequest()); // 1.Parse query and convert parse tree (CST) to abstract syntax tree (AST) ParseTree cst = parser.parse(request.getRequest()); Statement statement = diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 1227d5911fd..65323229162 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -7,7 +7,6 @@ import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; -import static org.opensearch.sql.ast.dsl.AstDSL.booleanLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.qualifiedName; import static org.opensearch.sql.calcite.utils.CalciteUtils.getOnlyForCalciteException; import static org.opensearch.sql.lang.PPLLangSpec.PPL_SPEC; @@ -46,7 +45,29 @@ import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.ast.EmptySourcePropagateVisitor; import org.opensearch.sql.ast.dsl.AstDSL; -import org.opensearch.sql.ast.expression.*; +import org.opensearch.sql.ast.expression.Alias; +import org.opensearch.sql.ast.expression.AllFieldsExcludeMeta; +import org.opensearch.sql.ast.expression.Argument; +import org.opensearch.sql.ast.expression.Argument.ArgumentMap; +import org.opensearch.sql.ast.expression.DataType; +import org.opensearch.sql.ast.expression.EqualTo; +import org.opensearch.sql.ast.expression.Field; +import org.opensearch.sql.ast.expression.Let; +import org.opensearch.sql.ast.expression.Literal; +import org.opensearch.sql.ast.expression.Map; +import org.opensearch.sql.ast.expression.ParseMethod; +import org.opensearch.sql.ast.expression.PatternMethod; +import org.opensearch.sql.ast.expression.PatternMode; +import org.opensearch.sql.ast.expression.QualifiedName; +import org.opensearch.sql.ast.expression.SearchAnd; +import org.opensearch.sql.ast.expression.SearchExpression; +import org.opensearch.sql.ast.expression.SearchGroup; +import org.opensearch.sql.ast.expression.Span; +import org.opensearch.sql.ast.expression.SpanUnit; +import org.opensearch.sql.ast.expression.UnresolvedArgument; +import org.opensearch.sql.ast.expression.UnresolvedExpression; +import org.opensearch.sql.ast.expression.WindowFrame; +import org.opensearch.sql.ast.expression.WindowFunction; import org.opensearch.sql.ast.tree.AD; import org.opensearch.sql.ast.tree.Aggregation; import org.opensearch.sql.ast.tree.Append; @@ -66,6 +87,7 @@ import org.opensearch.sql.ast.tree.Lookup; import org.opensearch.sql.ast.tree.ML; import org.opensearch.sql.ast.tree.MinSpanBin; +import org.opensearch.sql.ast.tree.Multisearch; import org.opensearch.sql.ast.tree.Parse; import org.opensearch.sql.ast.tree.Patterns; import org.opensearch.sql.ast.tree.Project; @@ -75,18 +97,22 @@ import org.opensearch.sql.ast.tree.Regex; import org.opensearch.sql.ast.tree.Relation; import org.opensearch.sql.ast.tree.Rename; +import org.opensearch.sql.ast.tree.Replace; +import org.opensearch.sql.ast.tree.ReplacePair; import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Rex; import org.opensearch.sql.ast.tree.SPath; import org.opensearch.sql.ast.tree.Search; import org.opensearch.sql.ast.tree.Sort; import org.opensearch.sql.ast.tree.SpanBin; +import org.opensearch.sql.ast.tree.StreamWindow; import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.Timechart; import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Window; +import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.setting.Settings.Key; import org.opensearch.sql.common.utils.StringUtils; @@ -379,6 +405,25 @@ public UnresolvedPlan visitRenameCommand(RenameCommandContext ctx) { .collect(Collectors.toList())); } + /** Replace command. */ + @Override + public UnresolvedPlan visitReplaceCommand(OpenSearchPPLParser.ReplaceCommandContext ctx) { + // Parse all replacement pairs + List replacePairs = + ctx.replacePair().stream().map(this::buildReplacePair).collect(Collectors.toList()); + + Set fieldList = getUniqueFieldSet(ctx.fieldList()); + + return new Replace(replacePairs, fieldList); + } + + /** Build a ReplacePair from parse context. */ + private ReplacePair buildReplacePair(OpenSearchPPLParser.ReplacePairContext ctx) { + Literal pattern = (Literal) internalVisitExpression(ctx.pattern); + Literal replacement = (Literal) internalVisitExpression(ctx.replacement); + return new ReplacePair(pattern, replacement); + } + /** Stats command. */ @Override public UnresolvedPlan visitStatsCommand(StatsCommandContext ctx) { @@ -424,6 +469,7 @@ public UnresolvedPlan visitStatsCommand(StatsCommandContext ctx) { return aggregation; } + /** Eventstats command. */ public UnresolvedPlan visitEventstatsCommand(OpenSearchPPLParser.EventstatsCommandContext ctx) { ImmutableList.Builder windownFunctionListBuilder = new ImmutableList.Builder<>(); @@ -445,6 +491,92 @@ public UnresolvedPlan visitEventstatsCommand(OpenSearchPPLParser.EventstatsComma return new Window(windownFunctionListBuilder.build()); } + /** Streamstats command. */ + public UnresolvedPlan visitStreamstatsCommand(OpenSearchPPLParser.StreamstatsCommandContext ctx) { + // 1. Parse arguments from the streamstats command + List argExprList = ArgumentFactory.getArgumentList(ctx); + ArgumentMap arguments = ArgumentMap.of(argExprList); + + // current, window and global from ArgumentFactory + boolean current = (Boolean) arguments.get("current").getValue(); + int window = (Integer) arguments.get("window").getValue(); + boolean global = (Boolean) arguments.get("global").getValue(); + + if (window < 0) { + throw new IllegalArgumentException("Window size must be >= 0, but got: " + window); + } + + // reset_before, reset_after + UnresolvedExpression resetBeforeExpr = + Optional.ofNullable(ctx.streamstatsArgs()) + .filter(args -> args.resetBeforeArg() != null && !args.resetBeforeArg().isEmpty()) + .map(args -> expressionBuilder.visit(args.resetBeforeArg(0).logicalExpression())) + .orElse(null); + + UnresolvedExpression resetAfterExpr = + Optional.ofNullable(ctx.streamstatsArgs()) + .filter(args -> args.resetAfterArg() != null && !args.resetAfterArg().isEmpty()) + .map(args -> expressionBuilder.visit(args.resetAfterArg(0).logicalExpression())) + .orElse(null); + + // 2.1 Build a WindowFrame from the provided arguments + WindowFrame frame = buildFrameFromArgs(current, window); + // 2.2 Build groupList + List groupList = getPartitionExprList(ctx.statsByClause()); + + // 3. Build each window function in the command + ImmutableList.Builder windowFunctionListBuilder = + new ImmutableList.Builder<>(); + + for (OpenSearchPPLParser.StreamstatsAggTermContext aggCtx : ctx.streamstatsAggTerm()) { + UnresolvedExpression windowFunction = internalVisitExpression(aggCtx.windowFunction()); + if (windowFunction instanceof WindowFunction wf) { + // Attach PARTITION BY clause expressions + wf.setPartitionByList(groupList); + // Inject the frame + wf.setWindowFrame(frame); + } + String name = + aggCtx.alias == null + ? getTextInQuery(aggCtx) + : StringUtils.unquoteIdentifier(aggCtx.alias.getText()); + Alias alias = new Alias(name, windowFunction); + windowFunctionListBuilder.add(alias); + } + + // 4. Build StreamWindow AST node + return new StreamWindow( + windowFunctionListBuilder.build(), + groupList, + current, + window, + global, + resetBeforeExpr, + resetAfterExpr); + } + + private WindowFrame buildFrameFromArgs(boolean current, int window) { + // Build the frame + if (window > 0) { + if (current) { + // N-1 PRECEDING to CURRENT ROW + return WindowFrame.of( + WindowFrame.FrameType.ROWS, (window - 1) + " PRECEDING", "CURRENT ROW"); + } else { + // N PRECEDING to 1 PRECEDING + return WindowFrame.of(WindowFrame.FrameType.ROWS, window + " PRECEDING", "1 PRECEDING"); + } + } else { + // Default: running total + if (current) { + return WindowFrame.toCurrentRow(); + } else { + // Default: running total excluding current row + return WindowFrame.of(WindowFrame.FrameType.ROWS, "UNBOUNDED PRECEDING", "1 PRECEDING"); + } + } + } + /** Dedup command. */ @Override public UnresolvedPlan visitDedupCommand(DedupCommandContext ctx) { @@ -563,29 +695,31 @@ public UnresolvedPlan visitBinCommand(BinCommandContext ctx) { @Override public UnresolvedPlan visitSortCommand(SortCommandContext ctx) { Integer count = ctx.count != null ? Math.max(0, Integer.parseInt(ctx.count.getText())) : 0; - boolean desc = ctx.DESC() != null || ctx.D() != null; + + List sortFieldContexts = ctx.sortbyClause().sortField(); + validateSortDirectionSyntax(sortFieldContexts); List sortFields = - ctx.sortbyClause().sortField().stream() + sortFieldContexts.stream() .map(sort -> (Field) internalVisitExpression(sort)) - .map(field -> desc ? reverseSortDirection(field) : field) .collect(Collectors.toList()); return new Sort(count, sortFields); } - private Field reverseSortDirection(Field field) { - List updatedArgs = - field.getFieldArgs().stream() - .map( - arg -> - "asc".equals(arg.getArgName()) - ? new Argument( - "asc", booleanLiteral(!((Boolean) arg.getValue().getValue()))) - : arg) - .collect(Collectors.toList()); + private void validateSortDirectionSyntax(List sortFields) { + boolean hasPrefix = + sortFields.stream() + .anyMatch(sortField -> sortField instanceof OpenSearchPPLParser.PrefixSortFieldContext); + boolean hasSuffix = + sortFields.stream() + .anyMatch(sortField -> sortField instanceof OpenSearchPPLParser.SuffixSortFieldContext); - return new Field(field.getField(), updatedArgs); + if (hasPrefix && hasSuffix) { + throw new SemanticCheckException( + "Cannot mix prefix (+/-) and suffix (asc/desc) sort direction syntax in the same" + + " command."); + } } /** Reverse command. */ @@ -598,39 +732,20 @@ public UnresolvedPlan visitReverseCommand(OpenSearchPPLParser.ReverseCommandCont @Override public UnresolvedPlan visitTimechartCommand(OpenSearchPPLParser.TimechartCommandContext ctx) { UnresolvedExpression binExpression = - AstDSL.span(AstDSL.field("@timestamp"), AstDSL.intLiteral(1), SpanUnit.of("m")); + AstDSL.span(AstDSL.referImplicitTimestampField(), AstDSL.intLiteral(1), SpanUnit.m); Integer limit = 10; Boolean useOther = true; // Process timechart parameters for (OpenSearchPPLParser.TimechartParameterContext paramCtx : ctx.timechartParameter()) { - if (paramCtx.spanClause() != null) { - binExpression = internalVisitExpression(paramCtx.spanClause()); - } else if (paramCtx.spanLiteral() != null) { - Literal literal = (Literal) internalVisitExpression(paramCtx.spanLiteral()); - binExpression = AstDSL.spanFromSpanLengthLiteral(AstDSL.field("@timestamp"), literal); - } else if (paramCtx.timechartArg() != null) { - OpenSearchPPLParser.TimechartArgContext argCtx = paramCtx.timechartArg(); - if (argCtx.LIMIT() != null && argCtx.integerLiteral() != null) { - limit = Integer.parseInt(argCtx.integerLiteral().getText()); - if (limit < 0) { - throw new IllegalArgumentException("Limit must be a non-negative number"); - } - } else if (argCtx.USEOTHER() != null) { - if (argCtx.booleanLiteral() != null) { - useOther = Boolean.parseBoolean(argCtx.booleanLiteral().getText()); - } else if (argCtx.ident() != null) { - String useOtherValue = argCtx.ident().getText().toLowerCase(); - if ("true".equals(useOtherValue) || "t".equals(useOtherValue)) { - useOther = true; - } else if ("false".equals(useOtherValue) || "f".equals(useOtherValue)) { - useOther = false; - } else { - throw new IllegalArgumentException( - "Invalid useOther value: " - + argCtx.ident().getText() - + ". Expected true/false or t/f"); - } - } + UnresolvedExpression param = internalVisitExpression(paramCtx); + if (param instanceof Span) { + binExpression = param; + } else if (param instanceof Literal literal) { + if (DataType.BOOLEAN.equals(literal.getType())) { + useOther = (Boolean) literal.getValue(); + } else if (DataType.INTEGER.equals(literal.getType()) + || DataType.LONG.equals(literal.getType())) { + limit = (Integer) literal.getValue(); } } } @@ -667,26 +782,43 @@ private List getFieldList(FieldListContext ctx) { .collect(Collectors.toList()); } - /** Rare command. */ - @Override - public UnresolvedPlan visitRareCommand(OpenSearchPPLParser.RareCommandContext ctx) { - List groupList = - ctx.byClause() == null ? emptyList() : getGroupByList(ctx.byClause()); - return new RareTopN( - CommandType.RARE, - ArgumentFactory.getArgumentList(ctx), - getFieldList(ctx.fieldList()), - groupList); + private Set getUniqueFieldSet(FieldListContext ctx) { + List fields = + ctx.fieldExpression().stream() + .map(field -> (Field) internalVisitExpression(field)) + .toList(); + + Set uniqueFields = new java.util.LinkedHashSet<>(fields); + + if (uniqueFields.size() < fields.size()) { + // Find duplicates for error message + Set seen = new HashSet<>(); + Set duplicates = + fields.stream() + .map(f -> f.getField().toString()) + .filter(name -> !seen.add(name)) + .collect(Collectors.toSet()); + + throw new IllegalArgumentException( + String.format("Duplicate fields [%s] in Replace command", String.join(", ", duplicates))); + } + + return uniqueFields; } - /** Top command. */ + /** Rare and Top commands. */ @Override - public UnresolvedPlan visitTopCommand(OpenSearchPPLParser.TopCommandContext ctx) { + public UnresolvedPlan visitRareTopCommand(OpenSearchPPLParser.RareTopCommandContext ctx) { List groupList = ctx.byClause() == null ? emptyList() : getGroupByList(ctx.byClause()); + Integer noOfResults = + ctx.number != null + ? (Integer) ((Literal) expressionBuilder.visitIntegerLiteral(ctx.number)).getValue() + : 10; return new RareTopN( - CommandType.TOP, - ArgumentFactory.getArgumentList(ctx), + ctx.TOP() != null ? CommandType.TOP : CommandType.RARE, + noOfResults, + ArgumentFactory.getArgumentList(ctx, settings), getFieldList(ctx.fieldList()), groupList); } @@ -956,6 +1088,25 @@ public UnresolvedPlan visitFillNullUsing(OpenSearchPPLParser.FillNullUsingContex return FillNull.ofVariousValue(replacementsBuilder.build()); } + /** fillnull command - value= syntax: fillnull value= field1 field2 ... */ + @Override + public UnresolvedPlan visitFillNullValueWithFields( + OpenSearchPPLParser.FillNullValueWithFieldsContext ctx) { + return FillNull.ofSameValue( + internalVisitExpression(ctx.replacement), + ctx.fieldList().fieldExpression().stream() + .map(f -> (Field) internalVisitExpression(f)) + .toList(), + true); + } + + /** fillnull command - value= syntax: fillnull value= */ + @Override + public UnresolvedPlan visitFillNullValueAllFields( + OpenSearchPPLParser.FillNullValueAllFieldsContext ctx) { + return FillNull.ofSameValue(internalVisitExpression(ctx.replacement), List.of(), true); + } + @Override public UnresolvedPlan visitFlattenCommand(OpenSearchPPLParser.FlattenCommandContext ctx) { Field field = (Field) internalVisitExpression(ctx.fieldExpression()); @@ -1021,6 +1172,26 @@ public UnresolvedPlan visitAppendCommand(OpenSearchPPLParser.AppendCommandContex return new Append(subsearch); } + @Override + public UnresolvedPlan visitMultisearchCommand(OpenSearchPPLParser.MultisearchCommandContext ctx) { + List subsearches = new ArrayList<>(); + + // Process each subsearch + for (OpenSearchPPLParser.SubSearchContext subsearchCtx : ctx.subSearch()) { + // Use the existing visitSubSearch logic + UnresolvedPlan fullSubsearch = visitSubSearch(subsearchCtx); + subsearches.add(fullSubsearch); + } + + // Validate minimum number of subsearches + if (subsearches.size() < 2) { + throw new SyntaxCheckException( + "Multisearch command requires at least two subsearches. Provided: " + subsearches.size()); + } + + return new Multisearch(subsearches); + } + @Override public UnresolvedPlan visitRexCommand(OpenSearchPPLParser.RexCommandContext ctx) { UnresolvedExpression field = internalVisitExpression(ctx.rexExpr().field); diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 20aec5b094a..4a5230d356e 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -21,6 +21,7 @@ import java.util.stream.Stream; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.RuleContext; +import org.antlr.v4.runtime.tree.ParseTree; import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.expression.*; import org.opensearch.sql.ast.expression.subquery.ExistsSubquery; @@ -30,6 +31,7 @@ import org.opensearch.sql.calcite.plan.OpenSearchConstants; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.utils.StringUtils; +import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.BinaryArithmeticContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.BooleanLiteralContext; @@ -60,13 +62,14 @@ import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.MultiFieldRelevanceFunctionContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.PatternMethodContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.PatternModeContext; +import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.PerFunctionCallContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.RenameFieldExpressionContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.SingleFieldRelevanceFunctionContext; -import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.SortFieldContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.SpanClauseContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.StatsFunctionCallContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.StringLiteralContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.TableSourceContext; +import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.TimechartCommandContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.WcFieldExpressionContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParserBaseVisitor; import org.opensearch.sql.ppl.utils.ArgumentFactory; @@ -223,20 +226,54 @@ public UnresolvedExpression visitRenameFieldExpression(RenameFieldExpressionCont } @Override - public UnresolvedExpression visitSortField(SortFieldContext ctx) { + public UnresolvedExpression visitPrefixSortField(OpenSearchPPLParser.PrefixSortFieldContext ctx) { + return buildSortField(ctx.sortFieldExpression(), ctx); + } + + @Override + public UnresolvedExpression visitSuffixSortField(OpenSearchPPLParser.SuffixSortFieldContext ctx) { + return buildSortField(ctx.sortFieldExpression(), ctx); + } + + @Override + public UnresolvedExpression visitDefaultSortField( + OpenSearchPPLParser.DefaultSortFieldContext ctx) { + return buildSortField(ctx.sortFieldExpression(), ctx); + } + + @Override + public UnresolvedExpression visitInvalidMixedSortField( + OpenSearchPPLParser.InvalidMixedSortFieldContext ctx) { + String prefixOperator = ctx.PLUS() != null ? "+" : "-"; + String suffixKeyword = + ctx.ASC() != null ? "asc" : ctx.A() != null ? "a" : ctx.DESC() != null ? "desc" : "d"; - UnresolvedExpression fieldExpression = - visit(ctx.sortFieldExpression().fieldExpression().qualifiedName()); + throw new SemanticCheckException( + String.format( + "Cannot use both prefix (%s) and suffix (%s) sort direction syntax on the same field. " + + "Use either '%s%s' or '%s %s', not both.", + prefixOperator, + suffixKeyword, + prefixOperator, + ctx.sortFieldExpression().getText(), + ctx.sortFieldExpression().getText(), + suffixKeyword)); + } + + private Field buildSortField( + OpenSearchPPLParser.SortFieldExpressionContext sortFieldExpr, + OpenSearchPPLParser.SortFieldContext parentCtx) { + UnresolvedExpression fieldExpression = visit(sortFieldExpr.fieldExpression().qualifiedName()); - if (ctx.sortFieldExpression().IP() != null) { + if (sortFieldExpr.IP() != null) { fieldExpression = new Cast(fieldExpression, AstDSL.stringLiteral("ip")); - } else if (ctx.sortFieldExpression().NUM() != null) { + } else if (sortFieldExpr.NUM() != null) { fieldExpression = new Cast(fieldExpression, AstDSL.stringLiteral("double")); - } else if (ctx.sortFieldExpression().STR() != null) { + } else if (sortFieldExpr.STR() != null) { fieldExpression = new Cast(fieldExpression, AstDSL.stringLiteral("string")); } // AUTO() case uses the field expression as-is - return new Field(fieldExpression, ArgumentFactory.getArgumentList(ctx)); + return new Field(fieldExpression, ArgumentFactory.getArgumentList(parentCtx)); } @Override @@ -547,6 +584,18 @@ private List timestampFunctionArguments( return args; } + @Override + public UnresolvedExpression visitPerFunctionCall(PerFunctionCallContext ctx) { + ParseTree parent = ctx.getParent(); + String perFuncName = ctx.perFunction().funcName.getText(); + if (!(parent instanceof TimechartCommandContext)) { + throw new SyntaxCheckException( + perFuncName + " function can only be used within timechart command"); + } + return buildAggregateFunction( + perFuncName, Collections.singletonList(ctx.perFunction().functionArg())); + } + /** Literal and value. */ @Override public UnresolvedExpression visitIdentsAsQualifiedName(IdentsAsQualifiedNameContext ctx) { @@ -621,7 +670,7 @@ public UnresolvedExpression visitSpanClause(SpanClauseContext ctx) { if (ctx.fieldExpression() != null) { fieldExpression = visit(ctx.fieldExpression()); } else { - fieldExpression = AstDSL.field("@timestamp"); + fieldExpression = AstDSL.referImplicitTimestampField(); } Literal literal = (Literal) visit(ctx.value); return AstDSL.spanFromSpanLengthLiteral(fieldExpression, literal); @@ -919,6 +968,47 @@ public UnresolvedExpression visitTimeModifierValue( return AstDSL.stringLiteral(osDateMathExpression); } + @Override + public UnresolvedExpression visitTimechartParameter( + OpenSearchPPLParser.TimechartParameterContext ctx) { + UnresolvedExpression timechartParameter; + if (ctx.SPAN() != null) { + // Convert span=1h to span(@timestamp, 1h) + Literal spanLiteral = (Literal) visit(ctx.spanLiteral()); + timechartParameter = + AstDSL.spanFromSpanLengthLiteral(AstDSL.referImplicitTimestampField(), spanLiteral); + } else if (ctx.LIMIT() != null) { + Literal limit = (Literal) visit(ctx.integerLiteral()); + if ((Integer) limit.getValue() < 0) { + throw new IllegalArgumentException("Limit must be a non-negative number"); + } + timechartParameter = limit; + } else if (ctx.USEOTHER() != null) { + UnresolvedExpression useOther; + if (ctx.booleanLiteral() != null) { + useOther = visit(ctx.booleanLiteral()); + } else if (ctx.ident() != null) { + QualifiedName ident = visitIdentifiers(List.of(ctx.ident())); + String useOtherValue = ident.toString(); + if ("true".equalsIgnoreCase(useOtherValue) || "t".equalsIgnoreCase(useOtherValue)) { + useOther = AstDSL.booleanLiteral(true); + } else if ("false".equalsIgnoreCase(useOtherValue) || "f".equalsIgnoreCase(useOtherValue)) { + useOther = AstDSL.booleanLiteral(false); + } else { + throw new IllegalArgumentException( + "Invalid useOther value: " + ctx.ident().getText() + ". Expected true/false or t/f"); + } + } else { + throw new IllegalArgumentException("value for useOther must be a boolean or identifier"); + } + timechartParameter = useOther; + } else { + throw new IllegalArgumentException( + String.format("A parameter of timechart must be a span, limit or useOther, got %s", ctx)); + } + return timechartParameter; + } + /** * Process time range expressions (EARLIEST='value' or LATEST='value') It creates a Comparison * filter like @timestamp >= timeModifierValue diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java index e1d892fdfce..acf204e8030 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java @@ -9,11 +9,13 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; import org.antlr.v4.runtime.ParserRuleContext; import org.opensearch.sql.ast.expression.Argument; import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.tree.Join; +import org.opensearch.sql.ast.tree.RareTopN; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.exception.SemanticCheckException; @@ -21,11 +23,13 @@ import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.BooleanLiteralContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DecimalLiteralContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DedupCommandContext; +import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DefaultSortFieldContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.FieldsCommandContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.IntegerLiteralContext; -import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.RareCommandContext; +import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.PrefixSortFieldContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.SortFieldContext; -import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.TopCommandContext; +import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.StreamstatsCommandContext; +import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.SuffixSortFieldContext; /** Util class to get all arguments as a list from the PPL command. */ public class ArgumentFactory { @@ -86,6 +90,25 @@ private static boolean legacyPreferred(Settings settings) { || Boolean.TRUE.equals(settings.getSettingValue(Settings.Key.PPL_SYNTAX_LEGACY_PREFERRED)); } + /** + * Get list of {@link Argument}. + * + * @param ctx StreamstatsCommandContext instance + * @return the list of arguments fetched from the streamstats command + */ + public static List getArgumentList(StreamstatsCommandContext ctx) { + return Arrays.asList( + ctx.streamstatsArgs().currentArg() != null && !ctx.streamstatsArgs().currentArg().isEmpty() + ? new Argument("current", getArgumentValue(ctx.streamstatsArgs().currentArg(0).current)) + : new Argument("current", new Literal(true, DataType.BOOLEAN)), + ctx.streamstatsArgs().windowArg() != null && !ctx.streamstatsArgs().windowArg().isEmpty() + ? new Argument("window", getArgumentValue(ctx.streamstatsArgs().windowArg(0).window)) + : new Argument("window", new Literal(0, DataType.INTEGER)), + ctx.streamstatsArgs().globalArg() != null && !ctx.streamstatsArgs().globalArg().isEmpty() + ? new Argument("global", getArgumentValue(ctx.streamstatsArgs().globalArg(0).global)) + : new Argument("global", new Literal(true, DataType.BOOLEAN))); + } + /** * Get list of {@link Argument}. * @@ -112,57 +135,101 @@ public static List getArgumentList(DedupCommandContext ctx) { * @return the list of arguments fetched from the sort field in sort command */ public static List getArgumentList(SortFieldContext ctx) { + if (ctx instanceof PrefixSortFieldContext) { + return getArgumentList((PrefixSortFieldContext) ctx); + } else if (ctx instanceof SuffixSortFieldContext) { + return getArgumentList((SuffixSortFieldContext) ctx); + } else { + return getArgumentList((DefaultSortFieldContext) ctx); + } + } + + /** + * Get list of {@link Argument} for prefix sort field (+/- syntax). + * + * @param ctx PrefixSortFieldContext instance + * @return the list of arguments fetched from the prefix sort field + */ + public static List getArgumentList(PrefixSortFieldContext ctx) { return Arrays.asList( ctx.MINUS() != null ? new Argument("asc", new Literal(false, DataType.BOOLEAN)) : new Argument("asc", new Literal(true, DataType.BOOLEAN)), - ctx.sortFieldExpression().AUTO() != null - ? new Argument("type", new Literal("auto", DataType.STRING)) - : ctx.sortFieldExpression().IP() != null - ? new Argument("type", new Literal("ip", DataType.STRING)) - : ctx.sortFieldExpression().NUM() != null - ? new Argument("type", new Literal("num", DataType.STRING)) - : ctx.sortFieldExpression().STR() != null - ? new Argument("type", new Literal("str", DataType.STRING)) - : new Argument("type", new Literal(null, DataType.NULL))); + getTypeArgument(ctx.sortFieldExpression())); } /** - * Get list of {@link Argument}. + * Get list of {@link Argument} for suffix sort field (asc/desc syntax). * - * @param ctx TopCommandContext instance - * @return the list of arguments fetched from the top command + * @param ctx SuffixSortFieldContext instance + * @return the list of arguments fetched from the suffix sort field */ - public static List getArgumentList(TopCommandContext ctx) { + public static List getArgumentList(SuffixSortFieldContext ctx) { return Arrays.asList( - ctx.number != null - ? new Argument("noOfResults", getArgumentValue(ctx.number)) - : new Argument("noOfResults", new Literal(10, DataType.INTEGER)), - ctx.countfield != null - ? new Argument("countField", getArgumentValue(ctx.countfield)) - : new Argument("countField", new Literal("count", DataType.STRING)), - ctx.showcount != null - ? new Argument("showCount", getArgumentValue(ctx.showcount)) - : new Argument("showCount", new Literal(true, DataType.BOOLEAN))); + (ctx.DESC() != null || ctx.D() != null) + ? new Argument("asc", new Literal(false, DataType.BOOLEAN)) + : new Argument("asc", new Literal(true, DataType.BOOLEAN)), + getTypeArgument(ctx.sortFieldExpression())); + } + + /** + * Get list of {@link Argument} for default sort field (no direction specified). + * + * @param ctx DefaultSortFieldContext instance + * @return the list of arguments fetched from the default sort field + */ + public static List getArgumentList(DefaultSortFieldContext ctx) { + return Arrays.asList( + new Argument("asc", new Literal(true, DataType.BOOLEAN)), + getTypeArgument(ctx.sortFieldExpression())); + } + + /** Helper method to get type argument from sortFieldExpression. */ + private static Argument getTypeArgument(OpenSearchPPLParser.SortFieldExpressionContext ctx) { + if (ctx.AUTO() != null) { + return new Argument("type", new Literal("auto", DataType.STRING)); + } else if (ctx.IP() != null) { + return new Argument("type", new Literal("ip", DataType.STRING)); + } else if (ctx.NUM() != null) { + return new Argument("type", new Literal("num", DataType.STRING)); + } else if (ctx.STR() != null) { + return new Argument("type", new Literal("str", DataType.STRING)); + } else { + return new Argument("type", new Literal(null, DataType.NULL)); + } } /** * Get list of {@link Argument}. * * @param ctx RareCommandContext instance + * @param settings Settings instance * @return the list of argument with default number of results for the rare command */ - public static List getArgumentList(RareCommandContext ctx) { - return Arrays.asList( - ctx.number != null - ? new Argument("noOfResults", getArgumentValue(ctx.number)) - : new Argument("noOfResults", new Literal(10, DataType.INTEGER)), - ctx.countfield != null - ? new Argument("countField", getArgumentValue(ctx.countfield)) - : new Argument("countField", new Literal("count", DataType.STRING)), - ctx.showcount != null - ? new Argument("showCount", getArgumentValue(ctx.showcount)) - : new Argument("showCount", new Literal(true, DataType.BOOLEAN))); + public static List getArgumentList( + OpenSearchPPLParser.RareTopCommandContext ctx, Settings settings) { + List list = new ArrayList<>(); + Optional opt = + ctx.rareTopOption().stream().filter(op -> op.countField != null).findFirst(); + list.add( + new Argument( + RareTopN.Option.countField.name(), + opt.isPresent() + ? getArgumentValue(opt.get().countField) + : new Literal("count", DataType.STRING))); + opt = ctx.rareTopOption().stream().filter(op -> op.showCount != null).findFirst(); + list.add( + new Argument( + RareTopN.Option.showCount.name(), + opt.isPresent() ? getArgumentValue(opt.get().showCount) : Literal.TRUE)); + opt = ctx.rareTopOption().stream().filter(op -> op.useNull != null).findFirst(); + list.add( + new Argument( + RareTopN.Option.useNull.name(), + opt.isPresent() + ? getArgumentValue(opt.get().useNull) + : legacyPreferred(settings) ? Literal.TRUE : Literal.FALSE)); + return list; } /** diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index eeca282cb9d..5b599ae162c 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -69,6 +69,7 @@ import org.opensearch.sql.ast.tree.Join; import org.opensearch.sql.ast.tree.Lookup; import org.opensearch.sql.ast.tree.MinSpanBin; +import org.opensearch.sql.ast.tree.Multisearch; import org.opensearch.sql.ast.tree.Parse; import org.opensearch.sql.ast.tree.Patterns; import org.opensearch.sql.ast.tree.Project; @@ -77,11 +78,14 @@ import org.opensearch.sql.ast.tree.Regex; import org.opensearch.sql.ast.tree.Relation; import org.opensearch.sql.ast.tree.Rename; +import org.opensearch.sql.ast.tree.Replace; import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Rex; +import org.opensearch.sql.ast.tree.SPath; import org.opensearch.sql.ast.tree.Search; import org.opensearch.sql.ast.tree.Sort; import org.opensearch.sql.ast.tree.SpanBin; +import org.opensearch.sql.ast.tree.StreamWindow; import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.Timechart; @@ -276,6 +280,30 @@ public String visitRename(Rename node, String context) { return StringUtils.format("%s | rename %s", child, renames); } + @Override + public String visitReplace(Replace node, String context) { + // Get the child query string + String child = node.getChild().get(0).accept(this, context); + + // Build pattern/replacement pairs string + String pairs = + node.getReplacePairs().stream() + .map( + pair -> + StringUtils.format( + "%s WITH %s", + visitExpression(pair.getPattern()), visitExpression(pair.getReplacement()))) + .collect(Collectors.joining(", ")); + + // Get field list + String fieldListStr = + " IN " + + node.getFieldList().stream().map(Field::toString).collect(Collectors.joining(", ")); + + // Build the replace command string + return StringUtils.format("%s | replace %s%s", child, pairs, fieldListStr); + } + /** Build {@link LogicalAggregation}. */ @Override public String visitAggregation(Aggregation node, String context) { @@ -350,19 +378,29 @@ public String visitWindow(Window node, String context) { child, String.join(" ", visitExpressionList(node.getWindowFunctionList())).trim()); } + @Override + public String visitStreamWindow(StreamWindow node, String context) { + String child = node.getChild().get(0).accept(this, context); + return StringUtils.format( + "%s | streamstats %s", + child, String.join(" ", visitExpressionList(node.getWindowFunctionList())).trim()); + } + /** Build {@link LogicalRareTopN}. */ @Override public String visitRareTopN(RareTopN node, String context) { final String child = node.getChild().get(0).accept(this, context); ArgumentMap arguments = ArgumentMap.of(node.getArguments()); - Integer noOfResults = (Integer) arguments.get("noOfResults").getValue(); - String countField = (String) arguments.get("countField").getValue(); - Boolean showCount = (Boolean) arguments.get("showCount").getValue(); + Integer noOfResults = node.getNoOfResults(); + String countField = (String) arguments.get(RareTopN.Option.countField.name()).getValue(); + Boolean showCount = (Boolean) arguments.get(RareTopN.Option.showCount.name()).getValue(); + Boolean useNull = (Boolean) arguments.get(RareTopN.Option.useNull.name()).getValue(); String fields = visitFieldList(node.getFields()); String group = visitExpressionList(node.getGroupExprList()); String options = isCalciteEnabled(settings) - ? StringUtils.format("countield='%s' showcount=%s ", countField, showCount) + ? StringUtils.format( + "countield='%s' showcount=%s usenull=%s ", countField, showCount, useNull) : ""; return StringUtils.format( "%s | %s %d %s%s", @@ -580,6 +618,36 @@ public String visitAppend(Append node, String context) { return StringUtils.format("%s | append [%s ]", child, subsearch); } + @Override + public String visitMultisearch(Multisearch node, String context) { + List anonymizedSubsearches = new ArrayList<>(); + + for (UnresolvedPlan subsearch : node.getSubsearches()) { + String anonymizedSubsearch = anonymizeData(subsearch); + anonymizedSubsearch = "search " + anonymizedSubsearch; + anonymizedSubsearch = + anonymizedSubsearch + .replaceAll("\\bsource=\\w+", "source=table") // Replace table names after source= + .replaceAll( + "\\b(?!source|fields|where|stats|head|tail|sort|eval|rename|multisearch|search|table|identifier|\\*\\*\\*)\\w+(?=\\s*[<>=!])", + "identifier") // Replace field names before operators + .replaceAll( + "\\b(?!source|fields|where|stats|head|tail|sort|eval|rename|multisearch|search|table|identifier|\\*\\*\\*)\\w+(?=\\s*,)", + "identifier") // Replace field names before commas + .replaceAll( + "fields" + + " \\+\\s*\\b(?!source|fields|where|stats|head|tail|sort|eval|rename|multisearch|search|table|identifier|\\*\\*\\*)\\w+", + "fields + identifier") // Replace field names after 'fields +' + .replaceAll( + "fields" + + " \\+\\s*identifier,\\s*\\b(?!source|fields|where|stats|head|tail|sort|eval|rename|multisearch|search|table|identifier|\\*\\*\\*)\\w+", + "fields + identifier,identifier"); // Handle multiple fields + anonymizedSubsearches.add(StringUtils.format("[%s]", anonymizedSubsearch)); + } + + return StringUtils.format("| multisearch %s", String.join(" ", anonymizedSubsearches)); + } + @Override public String visitValues(Values node, String context) { // In case legacy SQL relies on it, return empty to fail open anyway. @@ -610,28 +678,63 @@ private String visitExpression(UnresolvedExpression expression) { public String visitFillNull(FillNull node, String context) { String child = node.getChild().get(0).accept(this, context); List> fieldFills = node.getReplacementPairs(); + + // Check if using value= syntax (added in 3.4) + if (node.isUseValueSyntax()) { + if (fieldFills.isEmpty()) { + return StringUtils.format("%s | fillnull value=%s", child, MASK_LITERAL); + } + return StringUtils.format( + "%s | fillnull value=%s %s", + child, + MASK_LITERAL, + fieldFills.stream() + .map(n -> visitExpression(n.getLeft())) + .collect(Collectors.joining(" "))); + } + + // Distinguish between with...in and using based on whether all values are the same if (fieldFills.isEmpty()) { return StringUtils.format("%s | fillnull with %s", child, MASK_LITERAL); } final UnresolvedExpression firstReplacement = fieldFills.getFirst().getRight(); if (fieldFills.stream().allMatch(n -> firstReplacement == n.getRight())) { + // All fields use same replacement value -> with...in syntax return StringUtils.format( "%s | fillnull with %s in %s", child, MASK_LITERAL, - node.getReplacementPairs().stream() + fieldFills.stream() .map(n -> visitExpression(n.getLeft())) .collect(Collectors.joining(", "))); } else { + // Different replacement values per field -> using syntax return StringUtils.format( "%s | fillnull using %s", child, - node.getReplacementPairs().stream() + fieldFills.stream() .map(n -> StringUtils.format("%s = %s", visitExpression(n.getLeft()), MASK_LITERAL)) .collect(Collectors.joining(", "))); } } + @Override + public String visitSpath(SPath node, String context) { + String child = node.getChild().get(0).accept(this, context); + StringBuilder builder = new StringBuilder(); + builder.append(child).append(" | spath"); + if (node.getInField() != null) { + builder.append(" input=").append(MASK_COLUMN); + } + if (node.getOutField() != null) { + builder.append(" output=").append(MASK_COLUMN); + } + if (node.getPath() != null) { + builder.append(" path=").append(MASK_COLUMN); + } + return builder.toString(); + } + @Override public String visitPatterns(Patterns node, String context) { String child = node.getChild().get(0).accept(this, context); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java index e4d7f912d89..67403741a82 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java @@ -5,11 +5,7 @@ package org.opensearch.sql.ppl.antlr; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; import java.util.List; import org.antlr.v4.runtime.CommonTokenStream; @@ -99,6 +95,30 @@ public void testSearchFieldsCommandCrossClusterShouldPass() { assertNotEquals(null, tree); } + @Test + public void testPerSecondFunctionInTimechartShouldPass() { + ParseTree tree = new PPLSyntaxParser().parse("source=t | timechart per_second(a)"); + assertNotEquals(null, tree); + } + + @Test + public void testPerMinuteFunctionInTimechartShouldPass() { + ParseTree tree = new PPLSyntaxParser().parse("source=t | timechart per_minute(a)"); + assertNotEquals(null, tree); + } + + @Test + public void testPerHourFunctionInTimechartShouldPass() { + ParseTree tree = new PPLSyntaxParser().parse("source=t | timechart per_hour(a)"); + assertNotEquals(null, tree); + } + + @Test + public void testPerDayFunctionInTimechartShouldPass() { + ParseTree tree = new PPLSyntaxParser().parse("source=t | timechart per_day(a)"); + assertNotEquals(null, tree); + } + @Test public void testDynamicSourceClauseParseTreeStructure() { String query = "source=[myindex, logs, fieldIndex=\"test\", count=100]"; @@ -116,22 +136,22 @@ public void testDynamicSourceClauseParseTreeStructure() { searchFrom.fromClause().dynamicSourceClause(); // Verify source references size and text - assertEquals( - "Should have 2 source references", - 2, - dynamicSource.sourceReferences().sourceReference().size()); + assertEquals("Should have 2 source references", 2, dynamicSource.sourceReference().size()); assertEquals( "Source references text should match", - "myindex,logs", - dynamicSource.sourceReferences().getText()); + "myindex", + dynamicSource.sourceReference(0).getText()); - // Verify filter args size and text assertEquals( - "Should have 2 filter args", 2, dynamicSource.sourceFilterArgs().sourceFilterArg().size()); + "Source references text should match", "logs", dynamicSource.sourceReference(1).getText()); + // Verify filter args size and text + assertEquals("Should have 2 filter args", 2, dynamicSource.sourceFilterArg().size()); assertEquals( "Filter args text should match", - "fieldIndex=\"test\",count=100", - dynamicSource.sourceFilterArgs().getText()); + "fieldIndex=\"test\"", + dynamicSource.sourceFilterArg(0).getText()); + assertEquals( + "Filter args text should match", "count=100", dynamicSource.sourceFilterArg(1).getText()); } @Test @@ -149,22 +169,21 @@ public void testDynamicSourceWithComplexFilters() { searchFrom.fromClause().dynamicSourceClause(); // Verify source references + assertEquals("Should have 2 source references", 2, dynamicSource.sourceReference().size()); assertEquals( - "Should have 2 source references", - 2, - dynamicSource.sourceReferences().sourceReference().size()); + "First source reference text", "vpc.flow_logs", dynamicSource.sourceReference(0).getText()); assertEquals( - "Source references text", - "vpc.flow_logs,api.gateway", - dynamicSource.sourceReferences().getText()); + "Second source reference text", "api.gateway", dynamicSource.sourceReference(1).getText()); // Verify filter args + assertEquals("Should have 3 filter args", 3, dynamicSource.sourceFilterArg().size()); assertEquals( - "Should have 3 filter args", 3, dynamicSource.sourceFilterArgs().sourceFilterArg().size()); + "First filter arg text", + "region=\"us-east-1\"", + dynamicSource.sourceFilterArg(0).getText()); assertEquals( - "Filter args text", - "region=\"us-east-1\",env=\"prod\",count=5", - dynamicSource.sourceFilterArgs().getText()); + "Second filter arg text", "env=\"prod\"", dynamicSource.sourceFilterArg(1).getText()); + assertEquals("Third filter arg text", "count=5", dynamicSource.sourceFilterArg(2).getText()); } @Test @@ -180,24 +199,61 @@ public void testDynamicSourceWithSingleSource() { OpenSearchPPLParser.DynamicSourceClauseContext dynamicSource = searchFrom.fromClause().dynamicSourceClause(); + assertEquals("Should have 1 source reference", 1, dynamicSource.sourceReference().size()); + assertEquals("Source reference text", "ds:myindex", dynamicSource.sourceReference(0).getText()); + + assertEquals("Should have 1 filter arg", 1, dynamicSource.sourceFilterArg().size()); assertEquals( - "Should have 1 source reference", - 1, - dynamicSource.sourceReferences().sourceReference().size()); - assertEquals("Source reference text", "ds:myindex", dynamicSource.sourceReferences().getText()); + "Filter arg text", "fieldIndex=\"test\"", dynamicSource.sourceFilterArg(0).getText()); + } + @Test + public void testDynamicSourceWithoutSource() { + String query = "source=[fieldIndex=\"httpStatus\", region=\"us-west-2\"]"; + OpenSearchPPLLexer lexer = new OpenSearchPPLLexer(new CaseInsensitiveCharStream(query)); + OpenSearchPPLParser parser = new OpenSearchPPLParser(new CommonTokenStream(lexer)); + + OpenSearchPPLParser.RootContext root = parser.root(); + OpenSearchPPLParser.SearchFromContext searchFrom = + (OpenSearchPPLParser.SearchFromContext) + root.pplStatement().queryStatement().pplCommands().searchCommand(); + OpenSearchPPLParser.DynamicSourceClauseContext dynamicSource = + searchFrom.fromClause().dynamicSourceClause(); + assertEquals("Should have no source references", 0, dynamicSource.sourceReference().size()); + assertEquals("Should have 2 filter arg", 2, dynamicSource.sourceFilterArg().size()); assertEquals( - "Should have 1 filter arg", 1, dynamicSource.sourceFilterArgs().sourceFilterArg().size()); + "First filter arg text", + "fieldIndex=\"httpStatus\"", + dynamicSource.sourceFilterArg(0).getText()); assertEquals( - "Filter arg text", "fieldIndex=\"test\"", dynamicSource.sourceFilterArgs().getText()); + "Second filter arg text", + "region=\"us-west-2\"", + dynamicSource.sourceFilterArg(1).getText()); } @Test - public void testDynamicSourceRequiresAtLeastOneSource() { - // The grammar requires at least one source reference before optional filter args - // This test documents that behavior - exceptionRule.expect(RuntimeException.class); - new PPLSyntaxParser().parse("source=[fieldIndex=\"httpStatus\", region=\"us-west-2\"]"); + public void testDynamicSourceWithAllSources() { + String query = "source=[`*`, fieldIndex=\"httpStatus\", region=\"us-west-2\"]"; + OpenSearchPPLLexer lexer = new OpenSearchPPLLexer(new CaseInsensitiveCharStream(query)); + OpenSearchPPLParser parser = new OpenSearchPPLParser(new CommonTokenStream(lexer)); + + OpenSearchPPLParser.RootContext root = parser.root(); + OpenSearchPPLParser.SearchFromContext searchFrom = + (OpenSearchPPLParser.SearchFromContext) + root.pplStatement().queryStatement().pplCommands().searchCommand(); + OpenSearchPPLParser.DynamicSourceClauseContext dynamicSource = + searchFrom.fromClause().dynamicSourceClause(); + assertEquals("Should have 1 source reference", 1, dynamicSource.sourceReference().size()); + assertEquals("Source reference text", "`*`", dynamicSource.sourceReference(0).getText()); + assertEquals("Should have 2 filter arg", 2, dynamicSource.sourceFilterArg().size()); + assertEquals( + "First filter arg text", + "fieldIndex=\"httpStatus\"", + dynamicSource.sourceFilterArg(0).getText()); + assertEquals( + "Second filter arg text", + "region=\"us-west-2\"", + dynamicSource.sourceFilterArg(1).getText()); } @Test @@ -213,18 +269,16 @@ public void testDynamicSourceWithDottedNames() { OpenSearchPPLParser.DynamicSourceClauseContext dynamicSource = searchFrom.fromClause().dynamicSourceClause(); + assertEquals("Should have 2 source references", 2, dynamicSource.sourceReference().size()); assertEquals( - "Should have 2 source references", - 2, - dynamicSource.sourceReferences().sourceReference().size()); + "First source reference text", "vpc.flow_logs", dynamicSource.sourceReference(0).getText()); assertEquals( - "Source references text", - "vpc.flow_logs,api.gateway.logs", - dynamicSource.sourceReferences().getText()); + "Second source reference text", + "api.gateway.logs", + dynamicSource.sourceReference(1).getText()); - assertEquals( - "Should have 1 filter arg", 1, dynamicSource.sourceFilterArgs().sourceFilterArg().size()); - assertEquals("Filter arg text", "env=\"prod\"", dynamicSource.sourceFilterArgs().getText()); + assertEquals("Should have 1 filter arg", 1, dynamicSource.sourceFilterArg().size()); + assertEquals("Filter arg text", "env=\"prod\"", dynamicSource.sourceFilterArg(0).getText()); } @Test @@ -240,15 +294,11 @@ public void testDynamicSourceWithSimpleFilter() { OpenSearchPPLParser.DynamicSourceClauseContext dynamicSource = searchFrom.fromClause().dynamicSourceClause(); - assertEquals( - "Should have 1 source reference", - 1, - dynamicSource.sourceReferences().sourceReference().size()); - assertEquals("Source reference text", "logs", dynamicSource.sourceReferences().getText()); + assertEquals("Should have 1 source reference", 1, dynamicSource.sourceReference().size()); + assertEquals("Source reference text", "logs", dynamicSource.sourceReference(0).getText()); - assertEquals( - "Should have 1 filter arg", 1, dynamicSource.sourceFilterArgs().sourceFilterArg().size()); - assertEquals("Filter arg text", "count=100", dynamicSource.sourceFilterArgs().getText()); + assertEquals("Should have 1 filter arg", 1, dynamicSource.sourceFilterArg().size()); + assertEquals("Filter arg text", "count=100", dynamicSource.sourceFilterArg(0).getText()); } @Test @@ -269,12 +319,12 @@ public void testDynamicSourceWithInClause() { searchFrom.fromClause().dynamicSourceClause(); assertNotNull("Dynamic source should exist", dynamicSource); - assertNotNull("Filter args should exist", dynamicSource.sourceFilterArgs()); + assertNotNull("Filter args should exist", dynamicSource); // The IN clause should be parsed as a sourceFilterArg assertTrue( "Should have at least one filter arg with IN clause", - dynamicSource.sourceFilterArgs().sourceFilterArg().size() >= 1); + dynamicSource.sourceFilterArg().size() >= 1); } @Test @@ -693,6 +743,124 @@ public void testBlockCommentShouldPass() { """)); } + @Test + public void testDynamicSourceWithIntermixedSourcesAndFilters() { + // Test intermixed sources and filters in various orders + String query = "source=[myindex, region=\"us-east-1\", logs, count=100, api.gateway]"; + OpenSearchPPLLexer lexer = new OpenSearchPPLLexer(new CaseInsensitiveCharStream(query)); + OpenSearchPPLParser parser = new OpenSearchPPLParser(new CommonTokenStream(lexer)); + + OpenSearchPPLParser.RootContext root = parser.root(); + OpenSearchPPLParser.SearchFromContext searchFrom = + (OpenSearchPPLParser.SearchFromContext) + root.pplStatement().queryStatement().pplCommands().searchCommand(); + OpenSearchPPLParser.DynamicSourceClauseContext dynamicSource = + searchFrom.fromClause().dynamicSourceClause(); + + // Verify we have 3 source references + assertEquals("Should have 3 source references", 3, dynamicSource.sourceReference().size()); + assertEquals("First source reference", "myindex", dynamicSource.sourceReference(0).getText()); + assertEquals("Second source reference", "logs", dynamicSource.sourceReference(1).getText()); + assertEquals( + "Third source reference", "api.gateway", dynamicSource.sourceReference(2).getText()); + + // Verify we have 2 filter args + assertEquals("Should have 2 filter args", 2, dynamicSource.sourceFilterArg().size()); + assertEquals( + "First filter arg", "region=\"us-east-1\"", dynamicSource.sourceFilterArg(0).getText()); + assertEquals("Second filter arg", "count=100", dynamicSource.sourceFilterArg(1).getText()); + } + + @Test + public void testDynamicSourceStartingWithFilter() { + // Test starting with a filter argument instead of source reference + String query = "source=[region=\"us-west-1\", myindex, env=\"prod\", logs]"; + OpenSearchPPLLexer lexer = new OpenSearchPPLLexer(new CaseInsensitiveCharStream(query)); + OpenSearchPPLParser parser = new OpenSearchPPLParser(new CommonTokenStream(lexer)); + + OpenSearchPPLParser.RootContext root = parser.root(); + OpenSearchPPLParser.SearchFromContext searchFrom = + (OpenSearchPPLParser.SearchFromContext) + root.pplStatement().queryStatement().pplCommands().searchCommand(); + OpenSearchPPLParser.DynamicSourceClauseContext dynamicSource = + searchFrom.fromClause().dynamicSourceClause(); + + // Verify source references + assertEquals("Should have 2 source references", 2, dynamicSource.sourceReference().size()); + assertEquals("First source reference", "myindex", dynamicSource.sourceReference(0).getText()); + assertEquals("Second source reference", "logs", dynamicSource.sourceReference(1).getText()); + + // Verify filter args + assertEquals("Should have 2 filter args", 2, dynamicSource.sourceFilterArg().size()); + assertEquals( + "First filter arg", "region=\"us-west-1\"", dynamicSource.sourceFilterArg(0).getText()); + assertEquals("Second filter arg", "env=\"prod\"", dynamicSource.sourceFilterArg(1).getText()); + } + + @Test + public void testDynamicSourceAlternatingPattern() { + // Test alternating pattern of sources and filters + String query = + "source=[ds:index1, type=\"logs\", ds:index2, count=50, ds:index3, region=\"us-east-1\"]"; + OpenSearchPPLLexer lexer = new OpenSearchPPLLexer(new CaseInsensitiveCharStream(query)); + OpenSearchPPLParser parser = new OpenSearchPPLParser(new CommonTokenStream(lexer)); + + OpenSearchPPLParser.RootContext root = parser.root(); + OpenSearchPPLParser.SearchFromContext searchFrom = + (OpenSearchPPLParser.SearchFromContext) + root.pplStatement().queryStatement().pplCommands().searchCommand(); + OpenSearchPPLParser.DynamicSourceClauseContext dynamicSource = + searchFrom.fromClause().dynamicSourceClause(); + + // Verify source references + assertEquals("Should have 3 source references", 3, dynamicSource.sourceReference().size()); + assertEquals("First source reference", "ds:index1", dynamicSource.sourceReference(0).getText()); + assertEquals( + "Second source reference", "ds:index2", dynamicSource.sourceReference(1).getText()); + assertEquals("Third source reference", "ds:index3", dynamicSource.sourceReference(2).getText()); + + // Verify filter args + assertEquals("Should have 3 filter args", 3, dynamicSource.sourceFilterArg().size()); + assertEquals("First filter arg", "type=\"logs\"", dynamicSource.sourceFilterArg(0).getText()); + assertEquals("Second filter arg", "count=50", dynamicSource.sourceFilterArg(1).getText()); + assertEquals( + "Third filter arg", "region=\"us-east-1\"", dynamicSource.sourceFilterArg(2).getText()); + } + + @Test + public void testDynamicSourceWithComplexIntermixedIN() { + // Test intermixed with IN clause filter + String query = + "source=[logs, fieldIndex IN (\"status\", \"error\"), api.gateway, region=\"us-east-1\"]"; + OpenSearchPPLLexer lexer = new OpenSearchPPLLexer(new CaseInsensitiveCharStream(query)); + OpenSearchPPLParser parser = new OpenSearchPPLParser(new CommonTokenStream(lexer)); + + OpenSearchPPLParser.RootContext root = parser.root(); + assertNotNull("Query should parse successfully", root); + + OpenSearchPPLParser.SearchFromContext searchFrom = + (OpenSearchPPLParser.SearchFromContext) + root.pplStatement().queryStatement().pplCommands().searchCommand(); + OpenSearchPPLParser.DynamicSourceClauseContext dynamicSource = + searchFrom.fromClause().dynamicSourceClause(); + + assertNotNull("Dynamic source should exist", dynamicSource); + + // Verify source references + assertEquals("Should have 2 source references", 2, dynamicSource.sourceReference().size()); + assertEquals("First source reference", "logs", dynamicSource.sourceReference(0).getText()); + assertEquals( + "Second source reference", "api.gateway", dynamicSource.sourceReference(1).getText()); + + // Verify filter args - should have IN clause and region filter + assertEquals("Should have 2 filter args", 2, dynamicSource.sourceFilterArg().size()); + assertTrue( + "First filter should contain IN clause", + dynamicSource.sourceFilterArg(0).getText().contains("IN")); + assertEquals( + "Second filter arg", "region=\"us-east-1\"", dynamicSource.sourceFilterArg(1).getText()); + } + @Test public void testWhereCommand() { assertNotEquals(null, new PPLSyntaxParser().parse("SOURCE=test | WHERE x")); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java index 56041984c4d..9dd01b30df5 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java @@ -40,8 +40,8 @@ import org.opensearch.sql.ast.statement.Query; import org.opensearch.sql.calcite.CalcitePlanContext; import org.opensearch.sql.calcite.CalciteRelNodeVisitor; +import org.opensearch.sql.calcite.SysLimit; import org.opensearch.sql.common.setting.Settings; -import org.opensearch.sql.common.setting.Settings.Key; import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; @@ -69,6 +69,8 @@ public void init() { doReturn(true).when(settings).getSettingValue(Settings.Key.CALCITE_ENGINE_ENABLED); doReturn(true).when(settings).getSettingValue(Settings.Key.CALCITE_SUPPORT_ALL_JOIN_TYPES); doReturn(true).when(settings).getSettingValue(Settings.Key.PPL_SYNTAX_LEGACY_PREFERRED); + doReturn(-1).when(settings).getSettingValue(Settings.Key.PPL_JOIN_SUBSEARCH_MAXOUT); + doReturn(-1).when(settings).getSettingValue(Settings.Key.PPL_SUBSEARCH_MAXOUT); doReturn(false).when(dataSourceService).dataSourceExists(any()); } @@ -90,8 +92,7 @@ protected CalcitePlanContext createBuilderContext() { /** Creates a CalcitePlanContext with transformed config. */ private CalcitePlanContext createBuilderContext(UnaryOperator transform) { config.context(Contexts.of(transform.apply(RelBuilder.Config.DEFAULT))); - return CalcitePlanContext.create( - config.build(), settings.getSettingValue(Key.QUERY_SIZE_LIMIT), PPL); + return CalcitePlanContext.create(config.build(), SysLimit.fromSettings(settings), PPL); } /** Get the root RelNode of the given PPL query */ diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregateFunctionTypeTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregateFunctionTypeTest.java index 1e1109c256f..b363be9bee1 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregateFunctionTypeTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregateFunctionTypeTest.java @@ -17,48 +17,48 @@ public CalcitePPLAggregateFunctionTypeTest() { @Test public void testAvgWithWrongArgType() { verifyQueryThrowsException( - "source=EMP | stats avg(ENAME) as avg_name", - "Aggregation function AVG expects field type {[INTEGER]|[DOUBLE]}, but got [STRING]"); + "source=EMP | stats avg(HIREDATE) as avg_name", + "Aggregation function AVG expects field type {[INTEGER]|[DOUBLE]}, but got [DATE]"); } @Test public void testVarsampWithWrongArgType() { verifyQueryThrowsException( - "source=EMP | stats var_samp(ENAME) as varsamp_name", - "Aggregation function VARSAMP expects field type {[INTEGER]|[DOUBLE]}, but got [STRING]"); + "source=EMP | stats var_samp(HIREDATE) as varsamp_name", + "Aggregation function VARSAMP expects field type {[INTEGER]|[DOUBLE]}, but got [DATE]"); } @Test public void testVarpopWithWrongArgType() { verifyQueryThrowsException( - "source=EMP | stats var_pop(ENAME) as varpop_name", - "Aggregation function VARPOP expects field type {[INTEGER]|[DOUBLE]}, but got [STRING]"); + "source=EMP | stats var_pop(HIREDATE) as varpop_name", + "Aggregation function VARPOP expects field type {[INTEGER]|[DOUBLE]}, but got [DATE]"); } @Test public void testStddevSampWithWrongArgType() { verifyQueryThrowsException( - "source=EMP | stats stddev_samp(ENAME) as stddev_name", + "source=EMP | stats stddev_samp(HIREDATE) as stddev_name", "Aggregation function STDDEV_SAMP expects field type {[INTEGER]|[DOUBLE]}, but got" - + " [STRING]"); + + " [DATE]"); } @Test public void testStddevPopWithWrongArgType() { verifyQueryThrowsException( - "source=EMP | stats stddev_pop(ENAME) as stddev_name", + "source=EMP | stats stddev_pop(HIREDATE) as stddev_name", "Aggregation function STDDEV_POP expects field type {[INTEGER]|[DOUBLE]}, but got" - + " [STRING]"); + + " [DATE]"); } @Test public void testPercentileApproxWithWrongArgType() { // First argument should be numeric verifyQueryThrowsException( - "source=EMP | stats percentile_approx(ENAME, 50) as percentile", + "source=EMP | stats percentile_approx(HIREDATE, 50) as percentile", "Aggregation function PERCENTILE_APPROX expects field type and additional arguments" + " {[INTEGER,INTEGER]|[INTEGER,DOUBLE]|[DOUBLE,INTEGER]|[DOUBLE,DOUBLE]|[INTEGER,INTEGER,INTEGER]|[INTEGER,INTEGER,DOUBLE]|[INTEGER,DOUBLE,INTEGER]|[INTEGER,DOUBLE,DOUBLE]|[DOUBLE,INTEGER,INTEGER]|[DOUBLE,INTEGER,DOUBLE]|[DOUBLE,DOUBLE,INTEGER]|[DOUBLE,DOUBLE,DOUBLE]}," - + " but got [STRING,INTEGER]"); + + " but got [DATE,INTEGER]"); } @Test @@ -155,10 +155,10 @@ public void testPercentileWithMissingParametersThrowsException() { @Test public void testPercentileWithInvalidParameterTypesThrowsException() { verifyQueryThrowsException( - "source=EMP | stats percentile(EMPNO, 50, ENAME)", + "source=EMP | stats percentile(EMPNO, 50, HIREDATE)", "Aggregation function PERCENTILE_APPROX expects field type and additional arguments" + " {[INTEGER,INTEGER]|[INTEGER,DOUBLE]|[DOUBLE,INTEGER]|[DOUBLE,DOUBLE]|[INTEGER,INTEGER,INTEGER]|[INTEGER,INTEGER,DOUBLE]|[INTEGER,DOUBLE,INTEGER]|[INTEGER,DOUBLE,DOUBLE]|[DOUBLE,INTEGER,INTEGER]|[DOUBLE,INTEGER,DOUBLE]|[DOUBLE,DOUBLE,INTEGER]|[DOUBLE,DOUBLE,DOUBLE]}," - + " but got [SHORT,INTEGER,STRING]"); + + " but got [SHORT,INTEGER,DATE]"); } @Test diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java index 0e0068b5169..1446c7b0470 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java @@ -938,4 +938,55 @@ public void testMinOnTimeField() { String expectedSparkSql = "SELECT MIN(`HIREDATE`) `min_hire_date`\nFROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } + + @Test + public void testSortAggregationMetrics1() { + String ppl = "source=EMP | stats bucket_nullable=false avg(SAL) as avg by DEPTNO | sort - avg"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "" + + "LogicalSort(sort0=[$0], dir0=[DESC-nulls-last])\n" + + " LogicalProject(avg=[$1], DEPTNO=[$0])\n" + + " LogicalAggregate(group=[{0}], avg=[AVG($1)])\n" + + " LogicalProject(DEPTNO=[$7], SAL=[$5])\n" + + " LogicalFilter(condition=[IS NOT NULL($7)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = + "avg=2916.666666; DEPTNO=10\navg=2175.; DEPTNO=20\navg=1566.666666; DEPTNO=30\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT AVG(`SAL`) `avg`, `DEPTNO`\n" + + "FROM `scott`.`EMP`\n" + + "WHERE `DEPTNO` IS NOT NULL\n" + + "GROUP BY `DEPTNO`\n" + + "ORDER BY 1 DESC"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testSortAggregationMetrics2() { + String ppl = + "source=EMP | stats avg(SAL) as avg by span(HIREDATE, 1year) as hiredate_span | sort" + + " avg"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "" + + "LogicalSort(sort0=[$0], dir0=[ASC-nulls-first])\n" + + " LogicalProject(avg=[$1], hiredate_span=[$0])\n" + + " LogicalAggregate(group=[{1}], avg=[AVG($0)])\n" + + " LogicalProject(SAL=[$5], hiredate_span=[SPAN($4, 1, 'y')])\n" + + " LogicalFilter(condition=[IS NOT NULL($4)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT AVG(`SAL`) `avg`, `SPAN`(`HIREDATE`, 1, 'y') `hiredate_span`\n" + + "FROM `scott`.`EMP`\n" + + "WHERE `HIREDATE` IS NOT NULL\n" + + "GROUP BY `SPAN`(`HIREDATE`, 1, 'y')\n" + + "ORDER BY 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendTest.java index 614ed2ec32d..a163af186d5 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendTest.java @@ -9,6 +9,7 @@ import java.util.List; import org.apache.calcite.rel.RelNode; import org.apache.calcite.test.CalciteAssert; +import org.junit.Assert; import org.junit.Test; public class CalcitePPLAppendTest extends CalcitePPLAbstractTest { @@ -71,15 +72,16 @@ public void testAppendEmptySearchCommand() { @Test public void testAppendNested() { String ppl = - "source=EMP | append [ | where DEPTNO = 10 | append [ source=EMP | where DEPTNO = 20 ] ]"; + "source=EMP | fields ENAME, SAL | append [ | append [ source=EMP | where DEPTNO = 20 ] ]"; RelNode root = getRelNode(ppl); String expectedLogical = "LogicalUnion(all=[true])\n" - + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," - + " SAL=[$5], COMM=[$6], DEPTNO=[$7], EMPNO0=[null:SMALLINT])\n" + + " LogicalProject(ENAME=[$1], SAL=[$5], EMPNO=[null:SMALLINT], JOB=[null:VARCHAR(9)]," + + " MGR=[null:SMALLINT], HIREDATE=[null:DATE], COMM=[null:DECIMAL(7, 2)]," + + " DEPTNO=[null:TINYINT])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalProject(EMPNO=[null:SMALLINT], ENAME=[$1], JOB=[$2], MGR=[$3]," - + " HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], EMPNO0=[$0])\n" + + " LogicalProject(ENAME=[$1], SAL=[$5], EMPNO=[$0], JOB=[$2], MGR=[$3]," + + " HIREDATE=[$4], COMM=[$6], DEPTNO=[$7])\n" + " LogicalUnion(all=[true])\n" + " LogicalValues(tuples=[[]])\n" + " LogicalFilter(condition=[=($7, 20)])\n" @@ -88,12 +90,12 @@ public void testAppendNested() { verifyResultCount(root, 19); // 14 original table rows + 5 filtered subquery rows String expectedSparkSql = - "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, CAST(NULL AS" - + " SMALLINT) `EMPNO0`\n" + "SELECT `ENAME`, `SAL`, CAST(NULL AS SMALLINT) `EMPNO`, CAST(NULL AS STRING) `JOB`," + + " CAST(NULL AS SMALLINT) `MGR`, CAST(NULL AS DATE) `HIREDATE`, CAST(NULL AS" + + " DECIMAL(7, 2)) `COMM`, CAST(NULL AS TINYINT) `DEPTNO`\n" + "FROM `scott`.`EMP`\n" + "UNION ALL\n" - + "SELECT CAST(NULL AS SMALLINT) `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`," - + " `COMM`, `DEPTNO`, `EMPNO` `EMPNO0`\n" + + "SELECT `ENAME`, `SAL`, `EMPNO`, `JOB`, `MGR`, `HIREDATE`, `COMM`, `DEPTNO`\n" + "FROM (SELECT *\n" + "FROM (VALUES (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)) `t` (`EMPNO`," + " `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`)\n" @@ -109,61 +111,63 @@ public void testAppendNested() { public void testAppendEmptySourceWithJoin() { List emptySourceWithEmptySourceJoinPPLs = Arrays.asList( - "source=EMP | append [ | where DEPTNO = 10 | join on ENAME = DNAME DEPT ]", - "source=EMP | append [ | where DEPTNO = 10 | cross join on ENAME = DNAME DEPT ]", - "source=EMP | append [ | where DEPTNO = 10 | left join on ENAME = DNAME DEPT ]", - "source=EMP | append [ | where DEPTNO = 10 | semi join on ENAME = DNAME DEPT ]", - "source=EMP | append [ | where DEPTNO = 10 | anti join on ENAME = DNAME DEPT ]"); + "source=EMP | fields EMPNO, ENAME, JOB | append [ | where DEPTNO = 10 | join on ENAME" + + " = DNAME DEPT ]", + "source=EMP | fields EMPNO, ENAME, JOB | append [ | where DEPTNO = 10 | cross join on" + + " ENAME = DNAME DEPT ]", + "source=EMP | fields EMPNO, ENAME, JOB | append [ | where DEPTNO = 10 | left join on" + + " ENAME = DNAME DEPT ]", + "source=EMP | fields EMPNO, ENAME, JOB | append [ | where DEPTNO = 10 | semi join on" + + " ENAME = DNAME DEPT ]", + "source=EMP | fields EMPNO, ENAME, JOB | append [ | where DEPTNO = 10 | anti join on" + + " ENAME = DNAME DEPT ]"); for (String ppl : emptySourceWithEmptySourceJoinPPLs) { RelNode root = getRelNode(ppl); String expectedLogical = "LogicalUnion(all=[true])\n" - + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + " LogicalValues(tuples=[[]])\n"; verifyLogical(root, expectedLogical); verifyResultCount(root, 14); String expectedSparkSql = - "SELECT *\n" + "SELECT `EMPNO`, `ENAME`, `JOB`\n" + "FROM `scott`.`EMP`\n" + "UNION ALL\n" + "SELECT *\n" - + "FROM (VALUES (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)) `t` (`EMPNO`," - + " `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`)\n" + + "FROM (VALUES (NULL, NULL, NULL)) `t` (`EMPNO`, `ENAME`, `JOB`)\n" + "WHERE 1 = 0"; verifyPPLToSparkSQL(root, expectedSparkSql); } List emptySourceWithRightOrFullJoinPPLs = Arrays.asList( - "source=EMP | append [ | where DEPTNO = 10 | right join on ENAME = DNAME DEPT ]", - "source=EMP | append [ | where DEPTNO = 10 | full join on ENAME = DNAME DEPT ]"); + "source=EMP | fields EMPNO, ENAME, JOB | append [ | where DEPTNO = 10 | right join on" + + " ENAME = DNAME DEPT ]", + "source=EMP | fields EMPNO, ENAME, JOB | append [ | where DEPTNO = 10 | full join on" + + " ENAME = DNAME DEPT ]"); for (String ppl : emptySourceWithRightOrFullJoinPPLs) { RelNode root = getRelNode(ppl); String expectedLogical = "LogicalUnion(all=[true])\n" - + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," - + " SAL=[$5], COMM=[$6], DEPTNO=[$7], DEPTNO0=[null:TINYINT]," + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], DEPTNO=[null:TINYINT]," + " DNAME=[null:VARCHAR(14)], LOC=[null:VARCHAR(13)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" + " LogicalProject(EMPNO=[null:SMALLINT], ENAME=[null:VARCHAR(10)]," - + " JOB=[null:VARCHAR(9)], MGR=[null:SMALLINT], HIREDATE=[null:DATE]," - + " SAL=[null:DECIMAL(7, 2)], COMM=[null:DECIMAL(7, 2)], DEPTNO=[null:TINYINT]," - + " DEPTNO0=[$0], DNAME=[$1], LOC=[$2])\n" + + " JOB=[null:VARCHAR(9)], DEPTNO=[$0], DNAME=[$1], LOC=[$2])\n" + " LogicalTableScan(table=[[scott, DEPT]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, CAST(NULL AS" - + " TINYINT) `DEPTNO0`, CAST(NULL AS STRING) `DNAME`, CAST(NULL AS STRING) `LOC`\n" + "SELECT `EMPNO`, `ENAME`, `JOB`, CAST(NULL AS TINYINT) `DEPTNO`, CAST(NULL AS STRING)" + + " `DNAME`, CAST(NULL AS STRING) `LOC`\n" + "FROM `scott`.`EMP`\n" + "UNION ALL\n" + "SELECT CAST(NULL AS SMALLINT) `EMPNO`, CAST(NULL AS STRING) `ENAME`, CAST(NULL AS" - + " STRING) `JOB`, CAST(NULL AS SMALLINT) `MGR`, CAST(NULL AS DATE) `HIREDATE`," - + " CAST(NULL AS DECIMAL(7, 2)) `SAL`, CAST(NULL AS DECIMAL(7, 2)) `COMM`, CAST(NULL" - + " AS TINYINT) `DEPTNO`, `DEPTNO` `DEPTNO0`, `DNAME`, `LOC`\n" + + " STRING) `JOB`, `DEPTNO`, `DNAME`, `LOC`\n" + "FROM `scott`.`DEPT`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -172,15 +176,15 @@ public void testAppendEmptySourceWithJoin() { @Test public void testAppendDifferentIndex() { String ppl = - "source=EMP | fields EMPNO, DEPTNO | append [ source=DEPT | fields DEPTNO, DNAME | where" + "source=EMP | fields EMPNO, ENAME | append [ source=DEPT | fields DEPTNO, DNAME | where" + " DEPTNO = 20 ]"; RelNode root = getRelNode(ppl); String expectedLogical = "LogicalUnion(all=[true])\n" - + " LogicalProject(EMPNO=[$0], DEPTNO=[$7], DEPTNO0=[null:TINYINT]," + + " LogicalProject(EMPNO=[$0], ENAME=[$1], DEPTNO=[null:TINYINT]," + " DNAME=[null:VARCHAR(14)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalProject(EMPNO=[null:SMALLINT], DEPTNO=[null:TINYINT], DEPTNO0=[$0]," + + " LogicalProject(EMPNO=[null:SMALLINT], ENAME=[null:VARCHAR(10)], DEPTNO=[$0]," + " DNAME=[$1])\n" + " LogicalFilter(condition=[=($0, 20)])\n" + " LogicalProject(DEPTNO=[$0], DNAME=[$1])\n" @@ -188,11 +192,11 @@ public void testAppendDifferentIndex() { verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `EMPNO`, `DEPTNO`, CAST(NULL AS TINYINT) `DEPTNO0`, CAST(NULL AS STRING) `DNAME`\n" + "SELECT `EMPNO`, `ENAME`, CAST(NULL AS TINYINT) `DEPTNO`, CAST(NULL AS STRING) `DNAME`\n" + "FROM `scott`.`EMP`\n" + "UNION ALL\n" - + "SELECT CAST(NULL AS SMALLINT) `EMPNO`, CAST(NULL AS TINYINT) `DEPTNO`, `DEPTNO`" - + " `DEPTNO0`, `DNAME`\n" + + "SELECT CAST(NULL AS SMALLINT) `EMPNO`, CAST(NULL AS STRING) `ENAME`, `DEPTNO`," + + " `DNAME`\n" + "FROM (SELECT `DEPTNO`, `DNAME`\n" + "FROM `scott`.`DEPT`) `t0`\n" + "WHERE `DEPTNO` = 20"; @@ -227,22 +231,9 @@ public void testAppendWithMergedColumns() { public void testAppendWithConflictTypeColumn() { String ppl = "source=EMP | fields DEPTNO | append [ source=EMP | fields DEPTNO | eval DEPTNO = 20 ]"; - RelNode root = getRelNode(ppl); - String expectedLogical = - "LogicalUnion(all=[true])\n" - + " LogicalProject(DEPTNO=[$7], DEPTNO0=[null:INTEGER])\n" - + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalProject(DEPTNO=[null:TINYINT], DEPTNO0=[20])\n" - + " LogicalTableScan(table=[[scott, EMP]])\n"; - verifyLogical(root, expectedLogical); - verifyResultCount(root, 28); - - String expectedSparkSql = - "SELECT `DEPTNO`, CAST(NULL AS INTEGER) `DEPTNO0`\n" - + "FROM `scott`.`EMP`\n" - + "UNION ALL\n" - + "SELECT CAST(NULL AS TINYINT) `DEPTNO`, 20 `DEPTNO0`\n" - + "FROM `scott`.`EMP`"; - verifyPPLToSparkSQL(root, expectedSparkSql); + Exception exception = + Assert.assertThrows(IllegalArgumentException.class, () -> getRelNode(ppl)); + verifyErrorMessageContains( + exception, "Unable to process column 'DEPTNO' due to incompatible types:"); } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLBasicTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLBasicTest.java index 881a228a795..26783296f1c 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLBasicTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLBasicTest.java @@ -207,9 +207,7 @@ public void testFieldsMinusThenPlusShouldThrowException() { () -> { RelNode root = getRelNode(ppl); }); - assertThat( - e.getMessage(), - is("field [DEPTNO] not found; input fields are: [EMPNO, ENAME, JOB, MGR, HIREDATE, COMM]")); + assertThat(e.getMessage(), is("Field [DEPTNO] not found.")); } @Test @@ -319,7 +317,7 @@ public void testSortWithCountZero() { @Test public void testSortWithDescReversal() { - String ppl = "source=EMP | sort + DEPTNO, - SAL desc"; + String ppl = "source=EMP | sort DEPTNO desc, SAL"; RelNode root = getRelNode(ppl); String expectedLogical = "LogicalSort(sort0=[$7], sort1=[$5], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first])\n" @@ -329,7 +327,7 @@ public void testSortWithDescReversal() { @Test public void testSortWithDReversal() { - String ppl = "source=EMP | sort + DEPTNO, - SAL d"; + String ppl = "source=EMP | sort DEPTNO d, SAL"; RelNode root = getRelNode(ppl); String expectedLogical = "LogicalSort(sort0=[$7], sort1=[$5], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first])\n" diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLBinTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLBinTest.java index c940a750c28..ddb22092c99 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLBinTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLBinTest.java @@ -8,6 +8,7 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.test.CalciteAssert; import org.junit.Test; +import org.opensearch.sql.exception.SemanticCheckException; public class CalcitePPLBinTest extends CalcitePPLAbstractTest { @@ -118,4 +119,34 @@ public void testBinWithAligntime() { + " 3600 / 1) * 3600) `SYS_START`\n" + "FROM `scott`.`products_temporal`"); } + + @Test(expected = SemanticCheckException.class) + public void testBinWithMinspanOnNonNumericField() { + String ppl = "source=EMP | bin ENAME minspan=10"; + getRelNode(ppl); // Should throw SemanticCheckException + } + + @Test(expected = SemanticCheckException.class) + public void testBinWithSpanOnNonNumericField() { + String ppl = "source=EMP | bin JOB span=5"; + getRelNode(ppl); // Should throw SemanticCheckException + } + + @Test(expected = SemanticCheckException.class) + public void testBinWithBinsOnNonNumericField() { + String ppl = "source=EMP | bin ENAME bins=10"; + getRelNode(ppl); // Should throw SemanticCheckException + } + + @Test(expected = SemanticCheckException.class) + public void testBinWithStartEndOnNonNumericField() { + String ppl = "source=EMP | bin JOB start=1 end=10"; + getRelNode(ppl); // Should throw SemanticCheckException + } + + @Test(expected = SemanticCheckException.class) + public void testBinDefaultOnNonNumericField() { + String ppl = "source=EMP | bin ENAME"; + getRelNode(ppl); // Should throw SemanticCheckException + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEvalTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEvalTest.java index 95605b3a303..ea0194d68cf 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEvalTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEvalTest.java @@ -343,8 +343,7 @@ public void testComplexEvalCommands4() { () -> { RelNode root = getRelNode(ppl); }); - assertThat( - e.getMessage(), is("field [HIREDATE] not found; input fields are: [ENAME, col2, col3]")); + assertThat(e.getMessage(), is("Field [HIREDATE] not found.")); } @Test diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEventstatsTypeTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEventstatsTypeTest.java index a6535755435..24bd9ac18d0 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEventstatsTypeTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEventstatsTypeTest.java @@ -107,37 +107,37 @@ public void testLatestWithTooManyParametersThrowsException() { @Test public void testAvgWithWrongArgType() { verifyQueryThrowsException( - "source=EMP | eventstats avg(ENAME) as avg_name", - "Aggregation function AVG expects field type {[INTEGER]|[DOUBLE]}, but got [STRING]"); + "source=EMP | eventstats avg(HIREDATE) as avg_name", + "Aggregation function AVG expects field type {[INTEGER]|[DOUBLE]}, but got [DATE]"); } @Test public void testVarsampWithWrongArgType() { verifyQueryThrowsException( - "source=EMP | eventstats var_samp(ENAME) as varsamp_name", - "Aggregation function VARSAMP expects field type {[INTEGER]|[DOUBLE]}, but got [STRING]"); + "source=EMP | eventstats var_samp(HIREDATE) as varsamp_name", + "Aggregation function VARSAMP expects field type {[INTEGER]|[DOUBLE]}, but got [DATE]"); } @Test public void testVarpopWithWrongArgType() { verifyQueryThrowsException( - "source=EMP | eventstats var_pop(ENAME) as varpop_name", - "Aggregation function VARPOP expects field type {[INTEGER]|[DOUBLE]}, but got [STRING]"); + "source=EMP | eventstats var_pop(HIREDATE) as varpop_name", + "Aggregation function VARPOP expects field type {[INTEGER]|[DOUBLE]}, but got [DATE]"); } @Test public void testStddevSampWithWrongArgType() { verifyQueryThrowsException( - "source=EMP | eventstats stddev_samp(ENAME) as stddev_name", + "source=EMP | eventstats stddev_samp(HIREDATE) as stddev_name", "Aggregation function STDDEV_SAMP expects field type {[INTEGER]|[DOUBLE]}, but got" - + " [STRING]"); + + " [DATE]"); } @Test public void testStddevPopWithWrongArgType() { verifyQueryThrowsException( - "source=EMP | eventstats stddev_pop(ENAME) as stddev_name", + "source=EMP | eventstats stddev_pop(HIREDATE) as stddev_name", "Aggregation function STDDEV_POP expects field type {[INTEGER]|[DOUBLE]}, but got" - + " [STRING]"); + + " [DATE]"); } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExistsSubqueryTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExistsSubqueryTest.java index 717ad65ce27..76c280db92f 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExistsSubqueryTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExistsSubqueryTest.java @@ -5,9 +5,12 @@ package org.opensearch.sql.ppl.calcite; +import static org.mockito.Mockito.doReturn; + import org.apache.calcite.rel.RelNode; import org.apache.calcite.test.CalciteAssert; import org.junit.Test; +import org.opensearch.sql.common.setting.Settings; public class CalcitePPLExistsSubqueryTest extends CalcitePPLAbstractTest { public CalcitePPLExistsSubqueryTest() { @@ -501,4 +504,282 @@ public void testCorrelatedExistsSubqueryWithOverridingFields() { + "WHERE `t`.`DEPTNO` = `DEPTNO`)"; verifyPPLToSparkSQL(root, expectedSparkSql); } + + @Test + public void testSubsearchMaxOut1() { + doReturn(1).when(settings).getSettingValue(Settings.Key.PPL_SUBSEARCH_MAXOUT); + String ppl = + """ + source=EMP + | where exists [ + source=SALGRADE + | where EMP.SAL = HISAL + ] + | sort - EMPNO | fields EMPNO, ENAME + """; + RelNode root = getRelNode(ppl); + String expectedLogical = + "" + + "LogicalProject(EMPNO=[$0], ENAME=[$1])\n" + + " LogicalSort(sort0=[$0], dir0=[DESC-nulls-last])\n" + + " LogicalFilter(condition=[EXISTS({\n" + + "LogicalFilter(condition=[=($cor0.SAL, $2)])\n" + + " LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[1], type=[SUBSEARCH_MAXOUT])\n" + + " LogicalTableScan(table=[[scott, SALGRADE]])\n" + + "})], variablesSet=[[$cor0]])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`\n" + + "FROM `scott`.`EMP`\n" + + "WHERE EXISTS (SELECT *\n" + + "FROM (SELECT `GRADE`, `LOSAL`, `HISAL`\n" + + "FROM `scott`.`SALGRADE`\n" + + "ORDER BY `GRADE` NULLS LAST\n" + + "LIMIT 1) `t`\n" + + "WHERE `EMP`.`SAL` = `HISAL`)\n" + + "ORDER BY `EMPNO` DESC"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testSubsearchMaxOut2() { + doReturn(1).when(settings).getSettingValue(Settings.Key.PPL_SUBSEARCH_MAXOUT); + String ppl = + """ + source=EMP + | where exists [ + source=SALGRADE + | where EMP.SAL = HISAL and LOSAL > 1000.0 + ] + | sort - EMPNO | fields EMPNO, ENAME + """; + RelNode root = getRelNode(ppl); + String expectedLogical = + "" + + "LogicalProject(EMPNO=[$0], ENAME=[$1])\n" + + " LogicalSort(sort0=[$0], dir0=[DESC-nulls-last])\n" + + " LogicalFilter(condition=[EXISTS({\n" + + "LogicalFilter(condition=[=($cor0.SAL, $2)])\n" + + " LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[1], type=[SUBSEARCH_MAXOUT])\n" + + " LogicalFilter(condition=[>($1, 1000.0:DECIMAL(5, 1))])\n" + + " LogicalTableScan(table=[[scott, SALGRADE]])\n" + + "})], variablesSet=[[$cor0]])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`\n" + + "FROM `scott`.`EMP`\n" + + "WHERE EXISTS (SELECT *\n" + + "FROM (SELECT `GRADE`, `LOSAL`, `HISAL`\n" + + "FROM `scott`.`SALGRADE`\n" + + "WHERE `LOSAL` > 1000.0\n" + + "ORDER BY `GRADE` NULLS LAST\n" + + "LIMIT 1) `t0`\n" + + "WHERE `EMP`.`SAL` = `HISAL`)\n" + + "ORDER BY `EMPNO` DESC"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testSubsearchMaxOut3() { + doReturn(1).when(settings).getSettingValue(Settings.Key.PPL_SUBSEARCH_MAXOUT); + String ppl = + """ + source=EMP + | where exists [ + source=SALGRADE + | where EMP.SAL = HISAL + | eval LOSAL1 = LOSAL + | where LOSAL > 1000.0 + | sort - HISAL + ] + | sort - EMPNO | fields EMPNO, ENAME + """; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1])\n" + + " LogicalSort(sort0=[$0], dir0=[DESC-nulls-last])\n" + + " LogicalFilter(condition=[EXISTS({\n" + + "LogicalSort(sort0=[$2], dir0=[DESC-nulls-last])\n" + + " LogicalFilter(condition=[>($1, 1000.0:DECIMAL(5, 1))])\n" + + " LogicalProject(GRADE=[$0], LOSAL=[$1], HISAL=[$2], LOSAL1=[$1])\n" + + " LogicalFilter(condition=[=($cor0.SAL, $2)])\n" + + " LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[1]," + + " type=[SUBSEARCH_MAXOUT])\n" + + " LogicalTableScan(table=[[scott, SALGRADE]])\n" + + "})], variablesSet=[[$cor0]])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`\n" + + "FROM `scott`.`EMP`\n" + + "WHERE EXISTS (SELECT `GRADE`, `LOSAL`, `HISAL`, `LOSAL1`\n" + + "FROM (SELECT `GRADE`, `LOSAL`, `HISAL`, `LOSAL` `LOSAL1`\n" + + "FROM (SELECT `GRADE`, `LOSAL`, `HISAL`\n" + + "FROM `scott`.`SALGRADE`\n" + + "ORDER BY `GRADE` NULLS LAST\n" + + "LIMIT 1) `t`\n" + + "WHERE `EMP`.`SAL` = `HISAL`) `t1`\n" + + "WHERE `LOSAL` > 1000.0\n" + + "ORDER BY `HISAL` DESC)\n" + + "ORDER BY `EMPNO` DESC"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testSubsearchMaxOutUncorrelated1() { + doReturn(1).when(settings).getSettingValue(Settings.Key.PPL_SUBSEARCH_MAXOUT); + String ppl = + """ + source=EMP + | where exists [ + source=SALGRADE + | eval LOSAL1 = LOSAL + | where LOSAL > 1000.0 + | sort - HISAL + ] + | sort - EMPNO | fields EMPNO, ENAME + """; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1])\n" + + " LogicalSort(sort0=[$0], dir0=[DESC-nulls-last])\n" + + " LogicalFilter(condition=[EXISTS({\n" + + "LogicalSystemLimit(sort0=[$2], dir0=[DESC-nulls-last], fetch=[1]," + + " type=[SUBSEARCH_MAXOUT])\n" + + " LogicalSort(sort0=[$2], dir0=[DESC-nulls-last])\n" + + " LogicalFilter(condition=[>($1, 1000.0:DECIMAL(5, 1))])\n" + + " LogicalProject(GRADE=[$0], LOSAL=[$1], HISAL=[$2], LOSAL1=[$1])\n" + + " LogicalTableScan(table=[[scott, SALGRADE]])\n" + + "})], variablesSet=[[$cor0]])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`\n" + + "FROM `scott`.`EMP`\n" + + "WHERE EXISTS (SELECT `GRADE`, `LOSAL`, `HISAL`, `LOSAL1`\n" + + "FROM (SELECT `GRADE`, `LOSAL`, `HISAL`, `LOSAL1`\n" + + "FROM (SELECT `GRADE`, `LOSAL`, `HISAL`, `LOSAL` `LOSAL1`\n" + + "FROM `scott`.`SALGRADE`) `t`\n" + + "WHERE `LOSAL` > 1000.0\n" + + "ORDER BY `HISAL` DESC) `t1`\n" + + "ORDER BY `HISAL` DESC\n" + + "LIMIT 1)\n" + + "ORDER BY `EMPNO` DESC"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testSubsearchMaxOutUncorrelated2() { + doReturn(1).when(settings).getSettingValue(Settings.Key.PPL_SUBSEARCH_MAXOUT); + String ppl = + """ + source=EMP + | where exists [ + source=SALGRADE + | join type=left LOSAL SALGRADE + | eval LOSAL1 = LOSAL + | where LOSAL > 1000.0 + | sort - HISAL + ] + | sort - EMPNO | fields EMPNO, ENAME + """; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1])\n" + + " LogicalSort(sort0=[$0], dir0=[DESC-nulls-last])\n" + + " LogicalFilter(condition=[EXISTS({\n" + + "LogicalSystemLimit(sort0=[$2], dir0=[DESC-nulls-last], fetch=[1]," + + " type=[SUBSEARCH_MAXOUT])\n" + + " LogicalSort(sort0=[$2], dir0=[DESC-nulls-last])\n" + + " LogicalFilter(condition=[>($1, 1000.0:DECIMAL(5, 1))])\n" + + " LogicalProject(GRADE=[$3], LOSAL=[$4], HISAL=[$5], LOSAL1=[$4])\n" + + " LogicalJoin(condition=[=($1, $4)], joinType=[left])\n" + + " LogicalTableScan(table=[[scott, SALGRADE]])\n" + + " LogicalTableScan(table=[[scott, SALGRADE]])\n" + + "})], variablesSet=[[$cor0]])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`\n" + + "FROM `scott`.`EMP`\n" + + "WHERE EXISTS (SELECT `GRADE`, `LOSAL`, `HISAL`, `LOSAL1`\n" + + "FROM (SELECT `GRADE`, `LOSAL`, `HISAL`, `LOSAL1`\n" + + "FROM (SELECT `SALGRADE0`.`GRADE`, `SALGRADE0`.`LOSAL`, `SALGRADE0`.`HISAL`," + + " `SALGRADE0`.`LOSAL` `LOSAL1`\n" + + "FROM `scott`.`SALGRADE`\n" + + "LEFT JOIN `scott`.`SALGRADE` `SALGRADE0` ON `SALGRADE`.`LOSAL` =" + + " `SALGRADE0`.`LOSAL`) `t`\n" + + "WHERE `t`.`LOSAL` > 1000.0\n" + + "ORDER BY `HISAL` DESC) `t1`\n" + + "ORDER BY `HISAL` DESC\n" + + "LIMIT 1)\n" + + "ORDER BY `EMPNO` DESC"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testSubsearchMaxOutZeroMeansUnlimited() { + doReturn(0).when(settings).getSettingValue(Settings.Key.PPL_SUBSEARCH_MAXOUT); + String ppl = + """ + source=EMP + | where exists [ + source=SALGRADE + | where EMP.SAL = HISAL and LOSAL > 1000.0 + ] + | sort - EMPNO | fields EMPNO, ENAME + """; + RelNode root = getRelNode(ppl); + String expectedLogical = + "" + + "LogicalProject(EMPNO=[$0], ENAME=[$1])\n" + + " LogicalSort(sort0=[$0], dir0=[DESC-nulls-last])\n" + + " LogicalFilter(condition=[EXISTS({\n" + + "LogicalFilter(condition=[AND(=($cor0.SAL, $2), >($1, 1000.0:DECIMAL(5, 1)))])\n" + + " LogicalTableScan(table=[[scott, SALGRADE]])\n" + + "})], variablesSet=[[$cor0]])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`\n" + + "FROM `scott`.`EMP`\n" + + "WHERE EXISTS (SELECT *\n" + + "FROM `scott`.`SALGRADE`\n" + + "WHERE `EMP`.`SAL` = `HISAL` AND `LOSAL` > 1000.0)\n" + + "ORDER BY `EMPNO` DESC"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testSubsearchMaxOutNegativeMeansUnlimited() { + doReturn(-1).when(settings).getSettingValue(Settings.Key.PPL_SUBSEARCH_MAXOUT); + String ppl = + """ + source=EMP + | where exists [ + source=SALGRADE + | where EMP.SAL = HISAL and LOSAL > 1000.0 + ] + | sort - EMPNO | fields EMPNO, ENAME + """; + RelNode root = getRelNode(ppl); + String expectedLogical = + "" + + "LogicalProject(EMPNO=[$0], ENAME=[$1])\n" + + " LogicalSort(sort0=[$0], dir0=[DESC-nulls-last])\n" + + " LogicalFilter(condition=[EXISTS({\n" + + "LogicalFilter(condition=[AND(=($cor0.SAL, $2), >($1, 1000.0:DECIMAL(5, 1)))])\n" + + " LogicalTableScan(table=[[scott, SALGRADE]])\n" + + "})], variablesSet=[[$cor0]])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExpandTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExpandTest.java index 24f5d6ccc19..71ebce2c27e 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExpandTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExpandTest.java @@ -105,8 +105,8 @@ public void testExpand() { String expectedSparkSql = "SELECT `$cor0`.`DEPTNO`, `t00`.`EMPNOS`\n" + "FROM `scott`.`DEPT` `$cor0`,\n" - + "LATERAL UNNEST (SELECT `$cor0`.`EMPNOS`\n" - + "FROM (VALUES (0)) `t` (`ZERO`)) `t0` (`EMPNOS`) `t00`"; + + "LATERAL UNNEST((SELECT `$cor0`.`EMPNOS`\n" + + "FROM (VALUES (0)) `t` (`ZERO`))) `t00` (`EMPNOS`)"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -127,8 +127,8 @@ public void testExpandWithEval() { "SELECT `$cor0`.`DEPTNO`, `$cor0`.`EMPNOS`, `t10`.`employee_no`\n" + "FROM (SELECT `DEPTNO`, `EMPNOS`, `EMPNOS` `employee_no`\n" + "FROM `scott`.`DEPT`) `$cor0`,\n" - + "LATERAL UNNEST (SELECT `$cor0`.`employee_no`\n" - + "FROM (VALUES (0)) `t` (`ZERO`)) `t1` (`employee_no`) `t10`"; + + "LATERAL UNNEST((SELECT `$cor0`.`employee_no`\n" + + "FROM (VALUES (0)) `t` (`ZERO`))) `t10` (`employee_no`)"; verifyPPLToSparkSQL(root, expectedSparkSql); } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFillnullTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFillnullTest.java index 2a41d91e6ad..e664b4f21db 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFillnullTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFillnullTest.java @@ -141,4 +141,84 @@ public void testFillnullAll() { + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } + + @Test + public void testFillnullValueSyntaxWithFields() { + String ppl = "source=EMP | fillnull value=0 MGR COMM"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[COALESCE($3, 0)], HIREDATE=[$4]," + + " SAL=[$5], COMM=[COALESCE($6, 0)], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = + "EMPNO=7369; ENAME=SMITH; JOB=CLERK; MGR=7902; HIREDATE=1980-12-17; SAL=800.00; COMM=0;" + + " DEPTNO=20\n" + + "EMPNO=7499; ENAME=ALLEN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-20; SAL=1600.00;" + + " COMM=300.00; DEPTNO=30\n" + + "EMPNO=7521; ENAME=WARD; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-22; SAL=1250.00;" + + " COMM=500.00; DEPTNO=30\n" + + "EMPNO=7566; ENAME=JONES; JOB=MANAGER; MGR=7839; HIREDATE=1981-02-04; SAL=2975.00;" + + " COMM=0; DEPTNO=20\n" + + "EMPNO=7654; ENAME=MARTIN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-28; SAL=1250.00;" + + " COMM=1400.00; DEPTNO=30\n" + + "EMPNO=7698; ENAME=BLAKE; JOB=MANAGER; MGR=7839; HIREDATE=1981-01-05; SAL=2850.00;" + + " COMM=0; DEPTNO=30\n" + + "EMPNO=7782; ENAME=CLARK; JOB=MANAGER; MGR=7839; HIREDATE=1981-06-09; SAL=2450.00;" + + " COMM=0; DEPTNO=10\n" + + "EMPNO=7788; ENAME=SCOTT; JOB=ANALYST; MGR=7566; HIREDATE=1987-04-19; SAL=3000.00;" + + " COMM=0; DEPTNO=20\n" + + "EMPNO=7839; ENAME=KING; JOB=PRESIDENT; MGR=0; HIREDATE=1981-11-17; SAL=5000.00;" + + " COMM=0; DEPTNO=10\n" + + "EMPNO=7844; ENAME=TURNER; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-08; SAL=1500.00;" + + " COMM=0.00; DEPTNO=30\n" + + "EMPNO=7876; ENAME=ADAMS; JOB=CLERK; MGR=7788; HIREDATE=1987-05-23; SAL=1100.00;" + + " COMM=0; DEPTNO=20\n" + + "EMPNO=7900; ENAME=JAMES; JOB=CLERK; MGR=7698; HIREDATE=1981-12-03; SAL=950.00;" + + " COMM=0; DEPTNO=30\n" + + "EMPNO=7902; ENAME=FORD; JOB=ANALYST; MGR=7566; HIREDATE=1981-12-03; SAL=3000.00;" + + " COMM=0; DEPTNO=20\n" + + "EMPNO=7934; ENAME=MILLER; JOB=CLERK; MGR=7782; HIREDATE=1982-01-23; SAL=1300.00;" + + " COMM=0; DEPTNO=10\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, `JOB`, COALESCE(`MGR`, 0) `MGR`, `HIREDATE`, `SAL`," + + " COALESCE(`COMM`, 0) `COMM`, `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testFillnullValueSyntaxAllFields() { + String ppl = "source=EMP | fields EMPNO, MGR, SAL, COMM, DEPTNO | fillnull value=0"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], MGR=[COALESCE($3, 0)], SAL=[COALESCE($5, 0)]," + + " COMM=[COALESCE($6, 0)], DEPTNO=[COALESCE($7, 0)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = + "EMPNO=7369; MGR=7902; SAL=800.00; COMM=0; DEPTNO=20\n" + + "EMPNO=7499; MGR=7698; SAL=1600.00; COMM=300.00; DEPTNO=30\n" + + "EMPNO=7521; MGR=7698; SAL=1250.00; COMM=500.00; DEPTNO=30\n" + + "EMPNO=7566; MGR=7839; SAL=2975.00; COMM=0; DEPTNO=20\n" + + "EMPNO=7654; MGR=7698; SAL=1250.00; COMM=1400.00; DEPTNO=30\n" + + "EMPNO=7698; MGR=7839; SAL=2850.00; COMM=0; DEPTNO=30\n" + + "EMPNO=7782; MGR=7839; SAL=2450.00; COMM=0; DEPTNO=10\n" + + "EMPNO=7788; MGR=7566; SAL=3000.00; COMM=0; DEPTNO=20\n" + + "EMPNO=7839; MGR=0; SAL=5000.00; COMM=0; DEPTNO=10\n" + + "EMPNO=7844; MGR=7698; SAL=1500.00; COMM=0.00; DEPTNO=30\n" + + "EMPNO=7876; MGR=7788; SAL=1100.00; COMM=0; DEPTNO=20\n" + + "EMPNO=7900; MGR=7698; SAL=950.00; COMM=0; DEPTNO=30\n" + + "EMPNO=7902; MGR=7566; SAL=3000.00; COMM=0; DEPTNO=20\n" + + "EMPNO=7934; MGR=7782; SAL=1300.00; COMM=0; DEPTNO=10\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `EMPNO`, COALESCE(`MGR`, 0) `MGR`, COALESCE(`SAL`, 0) `SAL`, COALESCE(`COMM`, 0)" + + " `COMM`, COALESCE(`DEPTNO`, 0) `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFunctionTypeTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFunctionTypeTest.java index fc840169ee2..9513558952f 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFunctionTypeTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFunctionTypeTest.java @@ -45,12 +45,8 @@ public void testTimeDiffWithUdtInputType() { public void testComparisonWithDifferentType() { getRelNode("source=EMP | where EMPNO > 6 | fields ENAME"); getRelNode("source=EMP | where ENAME <= 'Jack' | fields ENAME"); - verifyQueryThrowsException( - "source=EMP | where ENAME < 6 | fields ENAME", - // Temporary fix for the error message as LESS function has two variants. Will remove - // [IP,IP] when merging the two variants. - "LESS function expects {[IP,IP],[COMPARABLE_TYPE,COMPARABLE_TYPE]}, but got" - + " [STRING,INTEGER]"); + // LogicalFilter(condition=[<(SAFE_CAST($1), 6.0E0)]) + getRelNode("source=EMP | where ENAME < 6 | fields ENAME"); } @Test @@ -151,8 +147,8 @@ public void testSha2WrongArgShouldThrow() { @Test public void testSqrtWithWrongArg() { verifyQueryThrowsException( - "source=EMP | head 1 | eval sqrt_name = sqrt(ENAME) | fields sqrt_name", - "SQRT function expects {[INTEGER]|[DOUBLE]}, but got [STRING]"); + "source=EMP | head 1 | eval sqrt_name = sqrt(HIREDATE) | fields sqrt_name", + "SQRT function expects {[INTEGER]|[DOUBLE]}, but got [DATE]"); } // Test UDF registered with PPL builtin operators: registerOperator(MOD, PPLBuiltinOperators.MOD); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLInSubqueryTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLInSubqueryTest.java index 99731859e28..3b4c2b72a27 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLInSubqueryTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLInSubqueryTest.java @@ -6,10 +6,12 @@ package org.opensearch.sql.ppl.calcite; import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.doReturn; import org.apache.calcite.rel.RelNode; import org.apache.calcite.test.CalciteAssert; import org.junit.Test; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.exception.SemanticCheckException; public class CalcitePPLInSubqueryTest extends CalcitePPLAbstractTest { @@ -255,4 +257,101 @@ public void failWhenNumOfColumnsNotMatchOutputOfSubquery() { """; assertThrows(SemanticCheckException.class, () -> getRelNode(more)); } + + @Test + public void testInCorrelatedSubquery() { + String ppl = + """ + source=EMP | where ENAME in [ + source=DEPT | where EMP.DEPTNO = DEPTNO and LOC = 'BOSTON'| fields DNAME + ] + | fields EMPNO, ENAME, DEPTNO + """; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], DEPTNO=[$7])\n" + + " LogicalFilter(condition=[IN($1, {\n" + + "LogicalProject(DNAME=[$1])\n" + + " LogicalFilter(condition=[AND(=($cor0.DEPTNO, $0), =($2, 'BOSTON':VARCHAR))])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + "})], variablesSet=[[$cor0]])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, `DEPTNO`\n" + + "FROM `scott`.`EMP`\n" + + "WHERE `ENAME` IN (SELECT `DNAME`\n" + + "FROM `scott`.`DEPT`\n" + + "WHERE `EMP`.`DEPTNO` = `DEPTNO` AND `LOC` = 'BOSTON')"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testSubsearchMaxOut() { + doReturn(1).when(settings).getSettingValue(Settings.Key.PPL_SUBSEARCH_MAXOUT); + String ppl = + """ + source=EMP | where DEPTNO in [ source=DEPT | fields DEPTNO ] + | sort - EMPNO | fields EMPNO, ENAME + """; + RelNode root = getRelNode(ppl); + String expectedLogical = + "" + + "LogicalProject(EMPNO=[$0], ENAME=[$1])\n" + + " LogicalSort(sort0=[$0], dir0=[DESC-nulls-last])\n" + + " LogicalFilter(condition=[IN($7, {\n" + + "LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[1], type=[SUBSEARCH_MAXOUT])\n" + + " LogicalProject(DEPTNO=[$0])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + "})], variablesSet=[[$cor0]])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`\n" + + "FROM `scott`.`EMP`\n" + + "WHERE `DEPTNO` IN (SELECT `DEPTNO`\n" + + "FROM `scott`.`DEPT`\n" + + "ORDER BY `DEPTNO` NULLS LAST\n" + + "LIMIT 1)\n" + + "ORDER BY `EMPNO` DESC"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testInCorrelatedSubqueryMaxOut() { + doReturn(1).when(settings).getSettingValue(Settings.Key.PPL_SUBSEARCH_MAXOUT); + String ppl = + """ + source=EMP | where ENAME in [ + source=DEPT | where EMP.DEPTNO = DEPTNO and LOC = 'BOSTON'| fields DNAME + ] + | fields EMPNO, ENAME, DEPTNO + """; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], DEPTNO=[$7])\n" + + " LogicalFilter(condition=[IN($1, {\n" + + "LogicalProject(DNAME=[$1])\n" + + " LogicalFilter(condition=[=($cor0.DEPTNO, $0)])\n" + + " LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[1], type=[SUBSEARCH_MAXOUT])\n" + + " LogicalFilter(condition=[=($2, 'BOSTON':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + "})], variablesSet=[[$cor0]])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, `DEPTNO`\n" + + "FROM `scott`.`EMP`\n" + + "WHERE `ENAME` IN (SELECT `DNAME`\n" + + "FROM (SELECT `DEPTNO`, `DNAME`, `LOC`\n" + + "FROM `scott`.`DEPT`\n" + + "WHERE `LOC` = 'BOSTON'\n" + + "ORDER BY `DEPTNO` NULLS LAST\n" + + "LIMIT 1) `t0`\n" + + "WHERE `EMP`.`DEPTNO` = `DEPTNO`)"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java index 773c020dc46..ff230540c93 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java @@ -1070,6 +1070,41 @@ public void testJoinWithMaxEqualsZero() { verifyPPLToSparkSQL(root, expectedSparkSql); } + @Test + public void testJoinSubsearchMaxOut() { + String ppl1 = "source=EMP | join type=inner max=0 DEPTNO DEPT"; + RelNode root1 = getRelNode(ppl1); + verifyResultCount(root1, 14); // no limit + String ppl2 = "source=EMP | inner join left=l right=r on l.DEPTNO=r.DEPTNO DEPT"; + RelNode root2 = getRelNode(ppl2); + verifyResultCount(root1, 14); // no limit for sql-like syntax + + doReturn(1).when(settings).getSettingValue(Settings.Key.PPL_JOIN_SUBSEARCH_MAXOUT); + root1 = getRelNode(ppl1); + verifyResultCount(root1, 3); // set maxout of subsesarch to 1 + root2 = getRelNode(ppl2); + verifyResultCount(root2, 3); // set maxout to 1 for sql-like syntax + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$8], DNAME=[$9], LOC=[$10])\n" + + " LogicalJoin(condition=[=($7, $8)], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[1]," + + " type=[JOIN_SUBSEARCH_MAXOUT])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + verifyLogical(root1, expectedLogical); + + String expectedSparkSql = + "SELECT `EMP`.`EMPNO`, `EMP`.`ENAME`, `EMP`.`JOB`, `EMP`.`MGR`, `EMP`.`HIREDATE`," + + " `EMP`.`SAL`, `EMP`.`COMM`, `t`.`DEPTNO`, `t`.`DNAME`, `t`.`LOC`\n" + + "FROM `scott`.`EMP`\n" + + "INNER JOIN (SELECT `DEPTNO`, `DNAME`, `LOC`\n" + + "FROM `scott`.`DEPT`\n" + + "ORDER BY `DEPTNO` NULLS LAST\n" + + "LIMIT 1) `t` ON `EMP`.`DEPTNO` = `t`.`DEPTNO`"; + verifyPPLToSparkSQL(root1, expectedSparkSql); + } + @Test public void testJoinWithMaxLessThanZero() { String ppl = "source=EMP | join type=outer max=-1 DEPTNO DEPT"; diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMathFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMathFunctionTest.java index 717ccad32f8..fff01427201 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMathFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMathFunctionTest.java @@ -124,7 +124,7 @@ public void testCrc32() { "LogicalProject(CRC32TEST=[CRC32('test':VARCHAR)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); - String expectedSparkSql = "SELECT `CRC32`('test') `CRC32TEST`\nFROM `scott`.`EMP`"; + String expectedSparkSql = "SELECT CRC32('test') `CRC32TEST`\nFROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -251,9 +251,10 @@ public void testModDecimal() { @Test public void testPi() { RelNode root = getRelNode("source=EMP | eval PI = pi() | fields PI"); - String expectedLogical = "LogicalProject(PI=[PI])\n LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedLogical = + "LogicalProject(PI=[PI()])\n LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); - String expectedSparkSql = "SELECT PI `PI`\nFROM `scott`.`EMP`"; + String expectedSparkSql = "SELECT PI() `PI`\nFROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMultisearchTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMultisearchTest.java new file mode 100644 index 00000000000..8746fe846e5 --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMultisearchTest.java @@ -0,0 +1,371 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.calcite; + +import com.google.common.collect.ImmutableList; +import java.sql.Timestamp; +import java.util.List; +import lombok.RequiredArgsConstructor; +import org.apache.calcite.DataContext; +import org.apache.calcite.config.CalciteConnectionConfig; +import org.apache.calcite.linq4j.Enumerable; +import org.apache.calcite.linq4j.Linq4j; +import org.apache.calcite.plan.RelTraitDef; +import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelProtoDataType; +import org.apache.calcite.schema.ScannableTable; +import org.apache.calcite.schema.Schema; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.Statistic; +import org.apache.calcite.schema.Statistics; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.test.CalciteAssert; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.Programs; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.junit.Test; + +public class CalcitePPLMultisearchTest extends CalcitePPLAbstractTest { + + public CalcitePPLMultisearchTest() { + super(CalciteAssert.SchemaSpec.SCOTT_WITH_TEMPORAL); + } + + @Override + protected Frameworks.ConfigBuilder config(CalciteAssert.SchemaSpec... schemaSpecs) { + final SchemaPlus rootSchema = Frameworks.createRootSchema(true); + final SchemaPlus schema = CalciteAssert.addSchema(rootSchema, schemaSpecs); + + // Add timestamp tables for multisearch testing with format matching time_test_data.json + ImmutableList timeData1 = + ImmutableList.of( + new Object[] { + Timestamp.valueOf("2025-08-01 03:47:41"), + 8762, + "A", + Timestamp.valueOf("2025-08-01 03:47:41") + }, + new Object[] { + Timestamp.valueOf("2025-08-01 01:14:11"), + 9015, + "B", + Timestamp.valueOf("2025-08-01 01:14:11") + }, + new Object[] { + Timestamp.valueOf("2025-07-31 23:40:33"), + 8676, + "A", + Timestamp.valueOf("2025-07-31 23:40:33") + }, + new Object[] { + Timestamp.valueOf("2025-07-31 21:07:03"), + 8490, + "B", + Timestamp.valueOf("2025-07-31 21:07:03") + }); + + ImmutableList timeData2 = + ImmutableList.of( + new Object[] { + Timestamp.valueOf("2025-08-01 04:00:00"), + 2001, + "E", + Timestamp.valueOf("2025-08-01 04:00:00") + }, + new Object[] { + Timestamp.valueOf("2025-08-01 02:30:00"), + 2002, + "F", + Timestamp.valueOf("2025-08-01 02:30:00") + }, + new Object[] { + Timestamp.valueOf("2025-08-01 01:00:00"), + 2003, + "E", + Timestamp.valueOf("2025-08-01 01:00:00") + }, + new Object[] { + Timestamp.valueOf("2025-07-31 22:15:00"), + 2004, + "F", + Timestamp.valueOf("2025-07-31 22:15:00") + }); + + schema.add("TIME_DATA1", new TimeDataTable(timeData1)); + schema.add("TIME_DATA2", new TimeDataTable(timeData2)); + + return Frameworks.newConfigBuilder() + .parserConfig(SqlParser.Config.DEFAULT) + .defaultSchema(schema) + .traitDefs((List) null) + .programs(Programs.heuristicJoinOrder(Programs.RULE_SET, true, 2)); + } + + @Test + public void testBasicMultisearch() { + String ppl = + "| multisearch " + + "[search source=EMP | where DEPTNO = 10] " + + "[search source=EMP | where DEPTNO = 20]"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalUnion(all=[true])\n" + + " LogicalFilter(condition=[=($7, 10)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalFilter(condition=[=($7, 20)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT *\n" + + "FROM `scott`.`EMP`\n" + + "WHERE `DEPTNO` = 10\n" + + "UNION ALL\n" + + "SELECT *\n" + + "FROM `scott`.`EMP`\n" + + "WHERE `DEPTNO` = 20"; + verifyPPLToSparkSQL(root, expectedSparkSql); + verifyResultCount(root, 8); + } + + @Test + public void testMultisearchCrossIndices() { + // Test multisearch with different tables (indices) + String ppl = + "| multisearch [search source=EMP | where DEPTNO = 10 | fields EMPNO, ENAME," + + " JOB] [search source=DEPT | where DEPTNO = 10 | fields DEPTNO, DNAME, LOC]"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalUnion(all=[true])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], DEPTNO=[null:TINYINT]," + + " DNAME=[null:VARCHAR(14)], LOC=[null:VARCHAR(13)])\n" + + " LogicalFilter(condition=[=($7, 10)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(EMPNO=[null:SMALLINT], ENAME=[null:VARCHAR(10)]," + + " JOB=[null:VARCHAR(9)], DEPTNO=[$0], DNAME=[$1], LOC=[$2])\n" + + " LogicalFilter(condition=[=($0, 10)])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, `JOB`, CAST(NULL AS TINYINT) `DEPTNO`, CAST(NULL AS STRING)" + + " `DNAME`, CAST(NULL AS STRING) `LOC`\n" + + "FROM `scott`.`EMP`\n" + + "WHERE `DEPTNO` = 10\n" + + "UNION ALL\n" + + "SELECT CAST(NULL AS SMALLINT) `EMPNO`, CAST(NULL AS STRING) `ENAME`, CAST(NULL AS" + + " STRING) `JOB`, `DEPTNO`, `DNAME`, `LOC`\n" + + "FROM `scott`.`DEPT`\n" + + "WHERE `DEPTNO` = 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + verifyResultCount(root, 4); // 3 employees + 1 department + } + + @Test + public void testMultisearchWithStats() { + String ppl = + "| multisearch " + + "[search source=EMP | where DEPTNO = 10 | eval type = \"accounting\"] " + + "[search source=EMP | where DEPTNO = 20 | eval type = \"research\"] " + + "| stats count by type"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(count=[$1], type=[$0])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(type=[$8])\n" + + " LogicalUnion(all=[true])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], type=['accounting':VARCHAR])\n" + + " LogicalFilter(condition=[=($7, 10)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], type=['research':VARCHAR])\n" + + " LogicalFilter(condition=[=($7, 20)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT COUNT(*) `count`, `type`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " 'accounting' `type`\n" + + "FROM `scott`.`EMP`\n" + + "WHERE `DEPTNO` = 10\n" + + "UNION ALL\n" + + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " 'research' `type`\n" + + "FROM `scott`.`EMP`\n" + + "WHERE `DEPTNO` = 20) `t3`\n" + + "GROUP BY `type`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + verifyResultCount(root, 2); + } + + @Test + public void testMultisearchThreeSubsearches() { + String ppl = + "| multisearch " + + "[search source=EMP | where DEPTNO = 10] " + + "[search source=EMP | where DEPTNO = 20] " + + "[search source=EMP | where DEPTNO = 30]"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalUnion(all=[true])\n" + + " LogicalFilter(condition=[=($7, 10)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalFilter(condition=[=($7, 20)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalFilter(condition=[=($7, 30)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT *\n" + + "FROM `scott`.`EMP`\n" + + "WHERE `DEPTNO` = 10\n" + + "UNION ALL\n" + + "SELECT *\n" + + "FROM `scott`.`EMP`\n" + + "WHERE `DEPTNO` = 20\n" + + "UNION ALL\n" + + "SELECT *\n" + + "FROM `scott`.`EMP`\n" + + "WHERE `DEPTNO` = 30"; + verifyPPLToSparkSQL(root, expectedSparkSql); + verifyResultCount(root, 14); + } + + // ======================================================================== + // Timestamp Interleaving Tests + // ======================================================================== + + @Test + public void testMultisearchTimestampInterleaving() { + String ppl = + "| multisearch " + + "[search source=TIME_DATA1 | where category IN (\"A\", \"B\")] " + + "[search source=TIME_DATA2 | where category IN (\"E\", \"F\")] " + + "| head 6"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(sort0=[$3], dir0=[DESC], fetch=[6])\n" + + " LogicalUnion(all=[true])\n" + + " LogicalFilter(condition=[SEARCH($2, Sarg['A':VARCHAR, 'B':VARCHAR]:VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, TIME_DATA1]])\n" + + " LogicalFilter(condition=[SEARCH($2, Sarg['E':VARCHAR, 'F':VARCHAR]:VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, TIME_DATA2]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT *\n" + + "FROM (SELECT *\n" + + "FROM `scott`.`TIME_DATA1`\n" + + "WHERE `category` IN ('A', 'B')\n" + + "UNION ALL\n" + + "SELECT *\n" + + "FROM `scott`.`TIME_DATA2`\n" + + "WHERE `category` IN ('E', 'F'))\n" + + "ORDER BY `@timestamp` DESC NULLS FIRST\n" + + "LIMIT 6"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testMultisearchWithTimestampFiltering() { + String ppl = + "| multisearch " + + "[search source=TIME_DATA1 | where @timestamp > \"2025-07-31 23:00:00\"] " + + "[search source=TIME_DATA2 | where @timestamp > \"2025-07-31 23:00:00\"] " + + "| sort @timestamp desc"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(sort0=[$3], dir0=[DESC-nulls-last])\n" + + " LogicalSort(sort0=[$3], dir0=[DESC])\n" + + " LogicalUnion(all=[true])\n" + + " LogicalFilter(condition=[>($3, TIMESTAMP('2025-07-31 23:00:00':VARCHAR))])\n" + + " LogicalTableScan(table=[[scott, TIME_DATA1]])\n" + + " LogicalFilter(condition=[>($3, TIMESTAMP('2025-07-31 23:00:00':VARCHAR))])\n" + + " LogicalTableScan(table=[[scott, TIME_DATA2]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT *\n" + + "FROM (SELECT `timestamp`, `value`, `category`, `@timestamp`\n" + + "FROM (SELECT *\n" + + "FROM `scott`.`TIME_DATA1`\n" + + "WHERE `@timestamp` > `TIMESTAMP`('2025-07-31 23:00:00')\n" + + "UNION ALL\n" + + "SELECT *\n" + + "FROM `scott`.`TIME_DATA2`\n" + + "WHERE `@timestamp` > `TIMESTAMP`('2025-07-31 23:00:00'))\n" + + "ORDER BY `@timestamp` DESC NULLS FIRST) `t2`\n" + + "ORDER BY `@timestamp` DESC"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + // ======================================================================== + // Custom Table Implementation for Timestamp Testing + // ======================================================================== + + /** Custom table implementation with timestamp fields for multisearch testing. */ + @RequiredArgsConstructor + static class TimeDataTable implements ScannableTable { + private final ImmutableList rows; + + protected final RelProtoDataType protoRowType = + factory -> + factory + .builder() + .add("timestamp", SqlTypeName.TIMESTAMP) + .nullable(true) + .add("value", SqlTypeName.INTEGER) + .nullable(true) + .add("category", SqlTypeName.VARCHAR) + .nullable(true) + .add("@timestamp", SqlTypeName.TIMESTAMP) + .nullable(true) + .build(); + + @Override + public Enumerable<@Nullable Object[]> scan(DataContext root) { + return Linq4j.asEnumerable(rows); + } + + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) { + return protoRowType.apply(typeFactory); + } + + @Override + public Statistic getStatistic() { + return Statistics.of(0d, ImmutableList.of(), RelCollations.createSingleton(0)); + } + + @Override + public Schema.TableType getJdbcTableType() { + return Schema.TableType.TABLE; + } + + @Override + public boolean isRolledUp(String column) { + return false; + } + + @Override + public boolean rolledUpColumnValidInsideAgg( + String column, + SqlCall call, + @Nullable SqlNode parent, + @Nullable CalciteConnectionConfig config) { + return false; + } + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLPatternsTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLPatternsTest.java index 9a88f25f270..d72c3b086cc 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLPatternsTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLPatternsTest.java @@ -351,11 +351,11 @@ public void testPatternsAggregationMode_NotShowNumberedToken_ForBrainMethod() { String expectedSparkSql = "SELECT SAFE_CAST(`t20`.`patterns_field`['pattern'] AS STRING) `patterns_field`," + " SAFE_CAST(`t20`.`patterns_field`['pattern_count'] AS BIGINT) `pattern_count`," - + " SAFE_CAST(`t20`.`patterns_field`['sample_logs'] AS VARCHAR ARRAY) `sample_logs`\n" + + " SAFE_CAST(`t20`.`patterns_field`['sample_logs'] AS ARRAY< STRING >) `sample_logs`\n" + "FROM (SELECT `pattern`(`ENAME`, 10, 100000, FALSE) `patterns_field`\n" + "FROM `scott`.`EMP`) `$cor0`,\n" - + "LATERAL UNNEST (SELECT `$cor0`.`patterns_field`\n" - + "FROM (VALUES (0)) `t` (`ZERO`)) `t2` (`patterns_field`) `t20`"; + + "LATERAL UNNEST((SELECT `$cor0`.`patterns_field`\n" + + "FROM (VALUES (0)) `t` (`ZERO`))) `t20` (`patterns_field`)"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -386,12 +386,12 @@ public void testPatternsAggregationMode_ShowNumberedToken_ForBrainMethod() { "SELECT SAFE_CAST(`t20`.`patterns_field`['pattern'] AS STRING) `patterns_field`," + " SAFE_CAST(`t20`.`patterns_field`['pattern_count'] AS BIGINT) `pattern_count`," + " SAFE_CAST(`t20`.`patterns_field`['tokens'] AS MAP< VARCHAR, VARCHAR ARRAY >)" - + " `tokens`, SAFE_CAST(`t20`.`patterns_field`['sample_logs'] AS VARCHAR ARRAY)" + + " `tokens`, SAFE_CAST(`t20`.`patterns_field`['sample_logs'] AS ARRAY< STRING >)" + " `sample_logs`\n" + "FROM (SELECT `pattern`(`ENAME`, 10, 100000, TRUE) `patterns_field`\n" + "FROM `scott`.`EMP`) `$cor0`,\n" - + "LATERAL UNNEST (SELECT `$cor0`.`patterns_field`\n" - + "FROM (VALUES (0)) `t` (`ZERO`)) `t2` (`patterns_field`) `t20`"; + + "LATERAL UNNEST((SELECT `$cor0`.`patterns_field`\n" + + "FROM (VALUES (0)) `t` (`ZERO`))) `t20` (`patterns_field`)"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -421,13 +421,13 @@ public void testPatternsAggregationModeWithGroupBy_NotShowNumberedToken_ForBrain String expectedSparkSql = "SELECT `$cor0`.`DEPTNO`, SAFE_CAST(`t20`.`patterns_field`['pattern'] AS STRING)" + " `patterns_field`, SAFE_CAST(`t20`.`patterns_field`['pattern_count'] AS BIGINT)" - + " `pattern_count`, SAFE_CAST(`t20`.`patterns_field`['sample_logs'] AS VARCHAR ARRAY)" - + " `sample_logs`\n" + + " `pattern_count`, SAFE_CAST(`t20`.`patterns_field`['sample_logs'] AS ARRAY< STRING" + + " >) `sample_logs`\n" + "FROM (SELECT `DEPTNO`, `pattern`(`ENAME`, 10, 100000, FALSE) `patterns_field`\n" + "FROM `scott`.`EMP`\n" + "GROUP BY `DEPTNO`) `$cor0`,\n" - + "LATERAL UNNEST (SELECT `$cor0`.`patterns_field`\n" - + "FROM (VALUES (0)) `t` (`ZERO`)) `t2` (`patterns_field`) `t20`"; + + "LATERAL UNNEST((SELECT `$cor0`.`patterns_field`\n" + + "FROM (VALUES (0)) `t` (`ZERO`))) `t20` (`patterns_field`)"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -460,12 +460,12 @@ public void testPatternsAggregationModeWithGroupBy_ShowNumberedToken_ForBrainMet + " `patterns_field`, SAFE_CAST(`t20`.`patterns_field`['pattern_count'] AS BIGINT)" + " `pattern_count`, SAFE_CAST(`t20`.`patterns_field`['tokens'] AS MAP< VARCHAR," + " VARCHAR ARRAY >) `tokens`, SAFE_CAST(`t20`.`patterns_field`['sample_logs'] AS" - + " VARCHAR ARRAY) `sample_logs`\n" + + " ARRAY< STRING >) `sample_logs`\n" + "FROM (SELECT `DEPTNO`, `pattern`(`ENAME`, 10, 100000, TRUE) `patterns_field`\n" + "FROM `scott`.`EMP`\n" + "GROUP BY `DEPTNO`) `$cor0`,\n" - + "LATERAL UNNEST (SELECT `$cor0`.`patterns_field`\n" - + "FROM (VALUES (0)) `t` (`ZERO`)) `t2` (`patterns_field`) `t20`"; + + "LATERAL UNNEST((SELECT `$cor0`.`patterns_field`\n" + + "FROM (VALUES (0)) `t` (`ZERO`))) `t20` (`patterns_field`)"; verifyPPLToSparkSQL(root, expectedSparkSql); } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLQualifiedNameResolutionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLQualifiedNameResolutionTest.java new file mode 100644 index 00000000000..17ea6f71dc6 --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLQualifiedNameResolutionTest.java @@ -0,0 +1,236 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.calcite; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.test.CalciteAssert; +import org.junit.Test; + +/** + * Tests for QualifiedNameResolver behavior in actual query planning scenarios. These tests verify + * that qualified name resolution works correctly during PPL query planning and produces the + * expected logical plans. + */ +public class CalcitePPLQualifiedNameResolutionTest extends CalcitePPLAbstractTest { + + public CalcitePPLQualifiedNameResolutionTest() { + super(CalciteAssert.SchemaSpec.SCOTT_WITH_TEMPORAL); + } + + @Test + public void testSimpleFieldReference() { + String ppl = "source=EMP | where DEPTNO = 20 | fields EMPNO, ENAME"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1])\n" + + " LogicalFilter(condition=[=($7, 20)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testQualifiedFieldReference() { + String ppl = "source=EMP as e | where e.DEPTNO = 20 | fields e.EMPNO, e.ENAME"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1])\n" + + " LogicalFilter(condition=[=($7, 20)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testJoinWithQualifiedNames() { + String ppl = + "source=EMP as e | join on e.DEPTNO = d.DEPTNO [ source=DEPT as d ] | fields e.EMPNO," + + " e.ENAME, d.DNAME"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], DNAME=[$9])\n" + + " LogicalJoin(condition=[=($7, $8)], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testJoinWithMixedQualifiedAndUnqualifiedNames() { + String ppl = + "source=EMP as e | join on e.DEPTNO = d.DEPTNO [ source=DEPT as d ] | fields e.EMPNO," + + " e.ENAME, d.DNAME"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], DNAME=[$9])\n" + + " LogicalJoin(condition=[=($7, $8)], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testJoinWithDuplicateFieldNames() { + String ppl = + "source=EMP as e | join on e.DEPTNO = d.DEPTNO [ source=DEPT as d ] | fields e.DEPTNO," + + " d.DEPTNO"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(DEPTNO=[$7], d.DEPTNO=[$8])\n" + + " LogicalJoin(condition=[=($7, $8)], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testComplexJoinCondition() { + String ppl = + "source=EMP as e | join on e.DEPTNO = d.DEPTNO AND e.SAL > 1000 [ source=DEPT as d ] |" + + " fields e.EMPNO, d.DNAME"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], DNAME=[$9])\n" + + " LogicalJoin(condition=[AND(=($7, $8), >($5, 1000))], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testRenamedFieldAccess() { + String ppl = + "source=EMP | rename DEPTNO as DEPT_ID | where DEPT_ID = 20 | fields EMPNO, DEPT_ID"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], DEPT_ID=[$7])\n" + + " LogicalFilter(condition=[=($7, 20)])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPT_ID=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testMultipleTableAliases() { + String ppl = + "source=EMP as emp | join on emp.DEPTNO = dept.DEPTNO [ source=DEPT as dept ] | where" + + " emp.SAL > 2000 | fields emp.EMPNO, dept.DNAME"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], DNAME=[$9])\n" + + " LogicalFilter(condition=[>($5, 2000)])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], dept.DEPTNO=[$8], DNAME=[$9], LOC=[$10])\n" + + " LogicalJoin(condition=[=($7, $8)], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testFieldAccessWithoutAlias() { + String ppl = "source=EMP | where DEPTNO = 20 AND SAL > 1000 | fields EMPNO, ENAME, SAL"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], SAL=[$5])\n" + + " LogicalFilter(condition=[AND(=($7, 20), >($5, 1000))])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testComplexExpressionWithQualifiedNames() { + String ppl = + "source=EMP as e | eval bonus = e.SAL * 0.1 | where e.DEPTNO = 20 | fields e.EMPNO, e.SAL," + + " bonus"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], SAL=[$5], bonus=[$8])\n" + + " LogicalFilter(condition=[=($7, 20)])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], bonus=[*($5, 0.1:DECIMAL(2, 1))])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testQualifiedNameInSortAndLimit() { + String ppl = "source=EMP as e | sort e.SAL desc | head 5 | fields e.EMPNO, e.ENAME, e.SAL"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], SAL=[$5])\n" + + " LogicalSort(sort0=[$5], dir0=[DESC-nulls-last], fetch=[5])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testQualifiedNameInAggregation() { + String ppl = + "source=EMP as e | stats avg(e.SAL) as avg_sal by e.DEPTNO | fields e.DEPTNO, avg_sal"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalAggregate(group=[{0}], avg_sal=[AVG($1)])\n" + + " LogicalProject(e.DEPTNO=[$7], SAL=[$5])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testMultipleJoinsWithQualifiedNames() { + String ppl = + "source=EMP as e | join on e.DEPTNO = d.DEPTNO [ source=DEPT as d ] | join on e.SAL >" + + " s.LOSAL AND e.SAL < s.HISAL [ source=SALGRADE as s ] | fields e.EMPNO, d.DNAME," + + " s.GRADE"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], DNAME=[$9], GRADE=[$11])\n" + + " LogicalJoin(condition=[AND(>($5, $12), <($5, $13))], joinType=[inner])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], d.DEPTNO=[$8], DNAME=[$9], LOC=[$10])\n" + + " LogicalJoin(condition=[=($7, $8)], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalTableScan(table=[[scott, SALGRADE]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testCorrelatedSubqueryWithQualifiedNames() { + String ppl = + "source=DEPT | where DEPTNO in [ source=EMP | where DEPTNO = DEPT.DEPTNO | fields DEPTNO ]" + + " | fields DNAME"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(DNAME=[$1])\n" + + " LogicalFilter(condition=[IN($0, {\n" + + "LogicalProject(DEPTNO=[$7])\n" + + " LogicalFilter(condition=[=($7, $cor0.DEPTNO)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + "})], variablesSet=[[$cor0]])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testFieldContainsDots() { + String ppl = + "source=DEPT | eval department.number = DEPTNO, DEPT.number = DEPTNO | where" + + " department.number in [ source=EMP | where DEPTNO = department.number | fields" + + " DEPTNO ] | fields DNAME, DEPT.number"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(DNAME=[$1], DEPT.number=[$4])\n" + + " LogicalFilter(condition=[IN($3, {\n" + + "LogicalProject(DEPTNO=[$7])\n" + + " LogicalFilter(condition=[=($7, $cor0.department.number)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + "})], variablesSet=[[$cor0]])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], department.number=[$0]," + + " DEPT.number=[$0])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + verifyLogical(root, expectedLogical); + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java index 23dab511671..2fcee849317 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java @@ -26,8 +26,8 @@ public void testRare() { String expectedLogical = "LogicalProject(JOB=[$0], count=[$1])\n" + " LogicalFilter(condition=[<=($2, 10)])\n" - + " LogicalProject(JOB=[$0], count=[$1], _row_number_=[ROW_NUMBER() OVER (ORDER BY" - + " $1)])\n" + + " LogicalProject(JOB=[$0], count=[$1], _row_number_rare_top_=[ROW_NUMBER() OVER" + + " (ORDER BY $1)])\n" + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + " LogicalProject(JOB=[$2])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; @@ -45,10 +45,10 @@ public void testRare() { String expectedSparkSql = "SELECT `JOB`, `count`\n" + "FROM (SELECT `JOB`, COUNT(*) `count`, ROW_NUMBER() OVER (ORDER BY COUNT(*) NULLS" - + " LAST) `_row_number_`\n" + + " LAST) `_row_number_rare_top_`\n" + "FROM `scott`.`EMP`\n" + "GROUP BY `JOB`) `t1`\n" - + "WHERE `_row_number_` <= 10"; + + "WHERE `_row_number_rare_top_` <= 10"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -59,8 +59,8 @@ public void testRareBy() { String expectedLogical = "LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2])\n" + " LogicalFilter(condition=[<=($3, 10)])\n" - + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2], _row_number_=[ROW_NUMBER()" - + " OVER (PARTITION BY $0 ORDER BY $2)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2)])\n" + " LogicalAggregate(group=[{0, 1}], count=[COUNT()])\n" + " LogicalProject(DEPTNO=[$7], JOB=[$2])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; @@ -82,10 +82,10 @@ public void testRareBy() { String expectedSparkSql = "SELECT `DEPTNO`, `JOB`, `count`\n" + "FROM (SELECT `DEPTNO`, `JOB`, COUNT(*) `count`, ROW_NUMBER() OVER (PARTITION BY" - + " `DEPTNO` ORDER BY COUNT(*) NULLS LAST) `_row_number_`\n" + + " `DEPTNO` ORDER BY COUNT(*) NULLS LAST) `_row_number_rare_top_`\n" + "FROM `scott`.`EMP`\n" + "GROUP BY `DEPTNO`, `JOB`) `t1`\n" - + "WHERE `_row_number_` <= 10"; + + "WHERE `_row_number_rare_top_` <= 10"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -96,8 +96,8 @@ public void testRareDisableShowCount() { String expectedLogical = "LogicalProject(DEPTNO=[$0], JOB=[$1])\n" + " LogicalFilter(condition=[<=($3, 10)])\n" - + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2], _row_number_=[ROW_NUMBER()" - + " OVER (PARTITION BY $0 ORDER BY $2)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2)])\n" + " LogicalAggregate(group=[{0, 1}], count=[COUNT()])\n" + " LogicalProject(DEPTNO=[$7], JOB=[$2])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; @@ -119,10 +119,10 @@ public void testRareDisableShowCount() { String expectedSparkSql = "SELECT `DEPTNO`, `JOB`\n" + "FROM (SELECT `DEPTNO`, `JOB`, COUNT(*) `count`, ROW_NUMBER() OVER (PARTITION BY" - + " `DEPTNO` ORDER BY COUNT(*) NULLS LAST) `_row_number_`\n" + + " `DEPTNO` ORDER BY COUNT(*) NULLS LAST) `_row_number_rare_top_`\n" + "FROM `scott`.`EMP`\n" + "GROUP BY `DEPTNO`, `JOB`) `t1`\n" - + "WHERE `_row_number_` <= 10"; + + "WHERE `_row_number_rare_top_` <= 10"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -133,8 +133,8 @@ public void testRareCountField() { String expectedLogical = "LogicalProject(DEPTNO=[$0], JOB=[$1], my_cnt=[$2])\n" + " LogicalFilter(condition=[<=($3, 10)])\n" - + " LogicalProject(DEPTNO=[$0], JOB=[$1], my_cnt=[$2], _row_number_=[ROW_NUMBER()" - + " OVER (PARTITION BY $0 ORDER BY $2)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], my_cnt=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2)])\n" + " LogicalAggregate(group=[{0, 1}], my_cnt=[COUNT()])\n" + " LogicalProject(DEPTNO=[$7], JOB=[$2])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; @@ -156,10 +156,49 @@ public void testRareCountField() { String expectedSparkSql = "SELECT `DEPTNO`, `JOB`, `my_cnt`\n" + "FROM (SELECT `DEPTNO`, `JOB`, COUNT(*) `my_cnt`, ROW_NUMBER() OVER (PARTITION BY" - + " `DEPTNO` ORDER BY COUNT(*) NULLS LAST) `_row_number_`\n" + + " `DEPTNO` ORDER BY COUNT(*) NULLS LAST) `_row_number_rare_top_`\n" + "FROM `scott`.`EMP`\n" + "GROUP BY `DEPTNO`, `JOB`) `t1`\n" - + "WHERE `_row_number_` <= 10"; + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testRareUseNullFalse() { + String ppl = "source=EMP | rare usenull=false JOB by DEPTNO"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2)])\n" + + " LogicalAggregate(group=[{0, 1}], count=[COUNT()])\n" + + " LogicalProject(DEPTNO=[$7], JOB=[$2])\n" + + " LogicalFilter(condition=[AND(IS NOT NULL($7), IS NOT NULL($2))])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "DEPTNO=20; JOB=MANAGER; count=1\n" + + "DEPTNO=20; JOB=CLERK; count=2\n" + + "DEPTNO=20; JOB=ANALYST; count=2\n" + + "DEPTNO=10; JOB=MANAGER; count=1\n" + + "DEPTNO=10; JOB=CLERK; count=1\n" + + "DEPTNO=10; JOB=PRESIDENT; count=1\n" + + "DEPTNO=30; JOB=MANAGER; count=1\n" + + "DEPTNO=30; JOB=CLERK; count=1\n" + + "DEPTNO=30; JOB=SALESMAN; count=4\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `DEPTNO`, `JOB`, `count`\n" + + "FROM (SELECT `DEPTNO`, `JOB`, COUNT(*) `count`, ROW_NUMBER() OVER (PARTITION BY" + + " `DEPTNO` ORDER BY COUNT(*) NULLS LAST) `_row_number_rare_top_`\n" + + "FROM `scott`.`EMP`\n" + + "WHERE `DEPTNO` IS NOT NULL AND `JOB` IS NOT NULL\n" + + "GROUP BY `DEPTNO`, `JOB`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -182,4 +221,187 @@ public void failWithDuplicatedName() { is("Field `DEPTNO` is existed, change the count field by setting countfield='xyz'")); } } + + @Test + public void testTop() { + String ppl = "source=EMP | top JOB"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(JOB=[$0], count=[$1])\n" + + " LogicalFilter(condition=[<=($2, 10)])\n" + + " LogicalProject(JOB=[$0], count=[$1], _row_number_rare_top_=[ROW_NUMBER() OVER" + + " (ORDER BY $1 DESC)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "JOB=SALESMAN; count=4\n" + + "JOB=CLERK; count=4\n" + + "JOB=MANAGER; count=3\n" + + "JOB=ANALYST; count=2\n" + + "JOB=PRESIDENT; count=1\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `count`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC" + + " NULLS FIRST) `_row_number_rare_top_`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testTopBy() { + String ppl = "source=EMP | top JOB by DEPTNO"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 DESC)])\n" + + " LogicalAggregate(group=[{0, 1}], count=[COUNT()])\n" + + " LogicalProject(DEPTNO=[$7], JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "DEPTNO=20; JOB=CLERK; count=2\n" + + "DEPTNO=20; JOB=ANALYST; count=2\n" + + "DEPTNO=20; JOB=MANAGER; count=1\n" + + "DEPTNO=10; JOB=MANAGER; count=1\n" + + "DEPTNO=10; JOB=CLERK; count=1\n" + + "DEPTNO=10; JOB=PRESIDENT; count=1\n" + + "DEPTNO=30; JOB=SALESMAN; count=4\n" + + "DEPTNO=30; JOB=MANAGER; count=1\n" + + "DEPTNO=30; JOB=CLERK; count=1\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `DEPTNO`, `JOB`, `count`\n" + + "FROM (SELECT `DEPTNO`, `JOB`, COUNT(*) `count`, ROW_NUMBER() OVER (PARTITION BY" + + " `DEPTNO` ORDER BY COUNT(*) DESC NULLS FIRST) `_row_number_rare_top_`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `DEPTNO`, `JOB`) `t1`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testTopDisableShowCount() { + String ppl = "source=EMP | top showcount=false JOB by DEPTNO"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(DEPTNO=[$0], JOB=[$1])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 DESC)])\n" + + " LogicalAggregate(group=[{0, 1}], count=[COUNT()])\n" + + " LogicalProject(DEPTNO=[$7], JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "DEPTNO=20; JOB=CLERK\n" + + "DEPTNO=20; JOB=ANALYST\n" + + "DEPTNO=20; JOB=MANAGER\n" + + "DEPTNO=10; JOB=MANAGER\n" + + "DEPTNO=10; JOB=CLERK\n" + + "DEPTNO=10; JOB=PRESIDENT\n" + + "DEPTNO=30; JOB=SALESMAN\n" + + "DEPTNO=30; JOB=MANAGER\n" + + "DEPTNO=30; JOB=CLERK\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `DEPTNO`, `JOB`\n" + + "FROM (SELECT `DEPTNO`, `JOB`, COUNT(*) `count`, ROW_NUMBER() OVER (PARTITION BY" + + " `DEPTNO` ORDER BY COUNT(*) DESC NULLS FIRST) `_row_number_rare_top_`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `DEPTNO`, `JOB`) `t1`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testTopCountField() { + String ppl = "source=EMP | top countfield='my_cnt' JOB by DEPTNO"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(DEPTNO=[$0], JOB=[$1], my_cnt=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], my_cnt=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 DESC)])\n" + + " LogicalAggregate(group=[{0, 1}], my_cnt=[COUNT()])\n" + + " LogicalProject(DEPTNO=[$7], JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "DEPTNO=20; JOB=CLERK; my_cnt=2\n" + + "DEPTNO=20; JOB=ANALYST; my_cnt=2\n" + + "DEPTNO=20; JOB=MANAGER; my_cnt=1\n" + + "DEPTNO=10; JOB=MANAGER; my_cnt=1\n" + + "DEPTNO=10; JOB=CLERK; my_cnt=1\n" + + "DEPTNO=10; JOB=PRESIDENT; my_cnt=1\n" + + "DEPTNO=30; JOB=SALESMAN; my_cnt=4\n" + + "DEPTNO=30; JOB=MANAGER; my_cnt=1\n" + + "DEPTNO=30; JOB=CLERK; my_cnt=1\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `DEPTNO`, `JOB`, `my_cnt`\n" + + "FROM (SELECT `DEPTNO`, `JOB`, COUNT(*) `my_cnt`, ROW_NUMBER() OVER (PARTITION BY" + + " `DEPTNO` ORDER BY COUNT(*) DESC NULLS FIRST) `_row_number_rare_top_`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `DEPTNO`, `JOB`) `t1`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testTopUseNullFalse() { + String ppl = "source=EMP | top usenull=false JOB by DEPTNO"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 DESC)])\n" + + " LogicalAggregate(group=[{0, 1}], count=[COUNT()])\n" + + " LogicalProject(DEPTNO=[$7], JOB=[$2])\n" + + " LogicalFilter(condition=[AND(IS NOT NULL($7), IS NOT NULL($2))])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "DEPTNO=20; JOB=CLERK; count=2\n" + + "DEPTNO=20; JOB=ANALYST; count=2\n" + + "DEPTNO=20; JOB=MANAGER; count=1\n" + + "DEPTNO=10; JOB=MANAGER; count=1\n" + + "DEPTNO=10; JOB=CLERK; count=1\n" + + "DEPTNO=10; JOB=PRESIDENT; count=1\n" + + "DEPTNO=30; JOB=SALESMAN; count=4\n" + + "DEPTNO=30; JOB=MANAGER; count=1\n" + + "DEPTNO=30; JOB=CLERK; count=1\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `DEPTNO`, `JOB`, `count`\n" + + "FROM (SELECT `DEPTNO`, `JOB`, COUNT(*) `count`, ROW_NUMBER() OVER (PARTITION BY" + + " `DEPTNO` ORDER BY COUNT(*) DESC NULLS FIRST) `_row_number_rare_top_`\n" + + "FROM `scott`.`EMP`\n" + + "WHERE `DEPTNO` IS NOT NULL AND `JOB` IS NOT NULL\n" + + "GROUP BY `DEPTNO`, `JOB`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLReplaceTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLReplaceTest.java new file mode 100644 index 00000000000..5f6f2beb76d --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLReplaceTest.java @@ -0,0 +1,400 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.calcite; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.test.CalciteAssert; +import org.junit.Test; +import org.opensearch.sql.common.antlr.SyntaxCheckException; + +public class CalcitePPLReplaceTest extends CalcitePPLAbstractTest { + + public CalcitePPLReplaceTest() { + super(CalciteAssert.SchemaSpec.SCOTT_WITH_TEMPORAL); + } + + @Test + public void testBasicReplace() { + String ppl = "source=EMP | replace \"CLERK\" WITH \"EMPLOYEE\" IN JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[REPLACE($2, 'CLERK':VARCHAR," + + " 'EMPLOYEE':VARCHAR)], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, REPLACE(`JOB`, 'CLERK', 'EMPLOYEE') `JOB`, `MGR`, `HIREDATE`," + + " `SAL`, `COMM`, `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + + String expectedResult = + "EMPNO=7369; ENAME=SMITH; JOB=EMPLOYEE; MGR=7902; HIREDATE=1980-12-17; SAL=800.00;" + + " COMM=null; DEPTNO=20\n" + + "EMPNO=7499; ENAME=ALLEN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-20; SAL=1600.00;" + + " COMM=300.00; DEPTNO=30\n" + + "EMPNO=7521; ENAME=WARD; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-22; SAL=1250.00;" + + " COMM=500.00; DEPTNO=30\n" + + "EMPNO=7566; ENAME=JONES; JOB=MANAGER; MGR=7839; HIREDATE=1981-02-04; SAL=2975.00;" + + " COMM=null; DEPTNO=20\n" + + "EMPNO=7654; ENAME=MARTIN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-28; SAL=1250.00;" + + " COMM=1400.00; DEPTNO=30\n" + + "EMPNO=7698; ENAME=BLAKE; JOB=MANAGER; MGR=7839; HIREDATE=1981-01-05; SAL=2850.00;" + + " COMM=null; DEPTNO=30\n" + + "EMPNO=7782; ENAME=CLARK; JOB=MANAGER; MGR=7839; HIREDATE=1981-06-09; SAL=2450.00;" + + " COMM=null; DEPTNO=10\n" + + "EMPNO=7788; ENAME=SCOTT; JOB=ANALYST; MGR=7566; HIREDATE=1987-04-19; SAL=3000.00;" + + " COMM=null; DEPTNO=20\n" + + "EMPNO=7839; ENAME=KING; JOB=PRESIDENT; MGR=null; HIREDATE=1981-11-17; SAL=5000.00;" + + " COMM=null; DEPTNO=10\n" + + "EMPNO=7844; ENAME=TURNER; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-08; SAL=1500.00;" + + " COMM=0.00; DEPTNO=30\n" + + "EMPNO=7876; ENAME=ADAMS; JOB=EMPLOYEE; MGR=7788; HIREDATE=1987-05-23; SAL=1100.00;" + + " COMM=null; DEPTNO=20\n" + + "EMPNO=7900; ENAME=JAMES; JOB=EMPLOYEE; MGR=7698; HIREDATE=1981-12-03; SAL=950.00;" + + " COMM=null; DEPTNO=30\n" + + "EMPNO=7902; ENAME=FORD; JOB=ANALYST; MGR=7566; HIREDATE=1981-12-03; SAL=3000.00;" + + " COMM=null; DEPTNO=20\n" + + "EMPNO=7934; ENAME=MILLER; JOB=EMPLOYEE; MGR=7782; HIREDATE=1982-01-23; SAL=1300.00;" + + " COMM=null; DEPTNO=10\n"; + + verifyResult(root, expectedResult); + } + + @Test + public void testMultipleFieldsReplace() { + String ppl = + "source=EMP | replace \"CLERK\" WITH \"EMPLOYEE\" IN JOB | replace \"20\" WITH \"RESEARCH\"" + + " IN DEPTNO"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[REPLACE($2, 'CLERK':VARCHAR," + + " 'EMPLOYEE':VARCHAR)], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6]," + + " DEPTNO=[REPLACE($7, '20':VARCHAR, 'RESEARCH':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, REPLACE(`JOB`, 'CLERK', 'EMPLOYEE') `JOB`, `MGR`, `HIREDATE`," + + " `SAL`, `COMM`, REPLACE(`DEPTNO`, '20', 'RESEARCH') `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testReplaceSameValueInMultipleFields() { + // In EMP table, both JOB and MGR fields contain numeric values + String ppl = "source=EMP | replace \"7839\" WITH \"CEO\" IN MGR, EMPNO"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(EMPNO=[REPLACE($0, '7839':VARCHAR, 'CEO':VARCHAR)], ENAME=[$1], JOB=[$2]," + + " MGR=[REPLACE($3, '7839':VARCHAR, 'CEO':VARCHAR)], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT REPLACE(`EMPNO`, '7839', 'CEO') `EMPNO`, `ENAME`, `JOB`," + + " REPLACE(`MGR`, '7839', 'CEO') `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testReplaceWithPipeline() { + String ppl = + "source=EMP | where JOB = 'CLERK' | replace \"CLERK\" WITH \"EMPLOYEE\" IN JOB | sort SAL"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalSort(sort0=[$5], dir0=[ASC-nulls-first])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[REPLACE($2, 'CLERK':VARCHAR," + + " 'EMPLOYEE':VARCHAR)], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalFilter(condition=[=($2, 'CLERK':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, REPLACE(`JOB`, 'CLERK', 'EMPLOYEE') `JOB`, `MGR`, `HIREDATE`," + + " `SAL`, `COMM`, `DEPTNO`\n" + + "FROM `scott`.`EMP`\n" + + "WHERE `JOB` = 'CLERK'\n" + + "ORDER BY `SAL`"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test(expected = SyntaxCheckException.class) + public void testReplaceWithoutWithKeywordShouldFail() { + String ppl = "source=EMP | replace \"CLERK\" \"EMPLOYEE\" IN JOB"; + getRelNode(ppl); + } + + @Test(expected = SyntaxCheckException.class) + public void testReplaceWithoutInKeywordShouldFail() { + String ppl = "source=EMP | replace \"CLERK\" WITH \"EMPLOYEE\" JOB"; + getRelNode(ppl); + } + + @Test(expected = SyntaxCheckException.class) + public void testReplaceWithExpressionShouldFail() { + String ppl = "source=EMP | replace EMPNO + 1 WITH \"EMPLOYEE\" IN JOB"; + getRelNode(ppl); + } + + @Test(expected = IllegalArgumentException.class) + public void testReplaceWithInvalidFieldShouldFail() { + String ppl = "source=EMP | replace \"CLERK\" WITH \"EMPLOYEE\" IN INVALID_FIELD"; + getRelNode(ppl); + } + + @Test(expected = IllegalArgumentException.class) + public void testReplaceWithMultipleInKeywordsShouldFail() { + String ppl = "source=EMP | replace \"CLERK\" WITH \"EMPLOYEE\" IN JOB IN ENAME"; + getRelNode(ppl); + } + + @Test(expected = SyntaxCheckException.class) + public void testReplaceWithMissingQuotesShouldFail() { + String ppl = "source=EMP | replace CLERK WITH EMPLOYEE IN JOB"; + getRelNode(ppl); + } + + @Test(expected = SyntaxCheckException.class) + public void testReplaceWithMissingReplacementValueShouldFail() { + String ppl = "source=EMP | replace \"CLERK\" WITH IN JOB"; + getRelNode(ppl); + } + + @Test + public void testReplaceWithEvalAndReplaceOnSameField() { + // Test verifies that in-place replacement works correctly when there are additional fields + // created by eval. The eval creates new_JOB, and replace modifies JOB in-place. + String ppl = + "source=EMP | eval new_JOB = 'existing' | replace \"CLERK\" WITH \"EMPLOYEE\" IN JOB"; + RelNode root = getRelNode(ppl); + + // With in-place replacement, JOB is modified and new_JOB remains as created by eval + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[REPLACE($2, 'CLERK':VARCHAR," + + " 'EMPLOYEE':VARCHAR)], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], new_JOB=['existing':VARCHAR])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, REPLACE(`JOB`, 'CLERK', 'EMPLOYEE') `JOB`, `MGR`, `HIREDATE`," + + " `SAL`, `COMM`, `DEPTNO`, 'existing' `new_JOB`\n" + + "FROM `scott`.`EMP`"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testReplaceWithMultiplePairs() { + // Test with multiple pattern/replacement pairs in a single command + String ppl = + "source=EMP | replace \"CLERK\" WITH \"EMPLOYEE\", \"MANAGER\" WITH \"SUPERVISOR\" IN JOB"; + RelNode root = getRelNode(ppl); + + // Should generate nested REPLACE calls: REPLACE(REPLACE(JOB, 'CLERK', 'EMPLOYEE'), 'MANAGER', + // 'SUPERVISOR') + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[REPLACE(REPLACE($2, 'CLERK':VARCHAR," + + " 'EMPLOYEE':VARCHAR), 'MANAGER':VARCHAR, 'SUPERVISOR':VARCHAR)], MGR=[$3]," + + " HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, REPLACE(REPLACE(`JOB`, 'CLERK', 'EMPLOYEE'), 'MANAGER'," + + " 'SUPERVISOR') `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testReplaceWithThreePairs() { + // Test with three pattern/replacement pairs + String ppl = + "source=EMP | replace \"CLERK\" WITH \"EMPLOYEE\", \"MANAGER\" WITH \"SUPERVISOR\"," + + " \"ANALYST\" WITH \"RESEARCHER\" IN JOB"; + RelNode root = getRelNode(ppl); + + // Should generate triple nested REPLACE calls + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[REPLACE(REPLACE(REPLACE($2," + + " 'CLERK':VARCHAR, 'EMPLOYEE':VARCHAR), 'MANAGER':VARCHAR, 'SUPERVISOR':VARCHAR)," + + " 'ANALYST':VARCHAR, 'RESEARCHER':VARCHAR)], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, REPLACE(REPLACE(REPLACE(`JOB`, 'CLERK', 'EMPLOYEE')," + + " 'MANAGER', 'SUPERVISOR'), 'ANALYST', 'RESEARCHER') `JOB`, `MGR`, `HIREDATE`," + + " `SAL`, `COMM`, `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testReplaceWithMultiplePairsOnMultipleFields() { + // Test with multiple pattern/replacement pairs applied to multiple fields + String ppl = + "source=EMP | replace \"CLERK\" WITH \"EMPLOYEE\", \"MANAGER\" WITH \"SUPERVISOR\" IN JOB," + + " ENAME"; + RelNode root = getRelNode(ppl); + + // Should apply the same nested REPLACE calls to both JOB and ENAME fields + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[REPLACE(REPLACE($1, 'CLERK':VARCHAR," + + " 'EMPLOYEE':VARCHAR), 'MANAGER':VARCHAR, 'SUPERVISOR':VARCHAR)]," + + " JOB=[REPLACE(REPLACE($2, 'CLERK':VARCHAR, 'EMPLOYEE':VARCHAR)," + + " 'MANAGER':VARCHAR, 'SUPERVISOR':VARCHAR)], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, REPLACE(REPLACE(`ENAME`, 'CLERK', 'EMPLOYEE'), 'MANAGER'," + + " 'SUPERVISOR') `ENAME`, REPLACE(REPLACE(`JOB`, 'CLERK', 'EMPLOYEE'), 'MANAGER'," + + " 'SUPERVISOR') `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testReplaceWithMultiplePairsSequentialApplication() { + // Test that replacements are applied sequentially + // This test demonstrates the order matters: if we have "20" WITH "30", "30" WITH "40" + // then "20" will become "30" first, then that "30" becomes "40", resulting in "40" + String ppl = "source=EMP | replace \"20\" WITH \"30\", \"30\" WITH \"40\" IN DEPTNO"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[REPLACE(REPLACE($7, '20':VARCHAR, '30':VARCHAR)," + + " '30':VARCHAR, '40':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`," + + " REPLACE(REPLACE(`DEPTNO`, '20', '30'), '30', '40') `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test(expected = SyntaxCheckException.class) + public void testReplaceWithMultiplePairsMissingWithKeywordShouldFail() { + // Missing WITH keyword between pairs + String ppl = + "source=EMP | replace \"CLERK\" \"EMPLOYEE\", \"MANAGER\" WITH \"SUPERVISOR\" IN JOB"; + getRelNode(ppl); + } + + @Test(expected = SyntaxCheckException.class) + public void testReplaceWithMultiplePairsTrailingCommaShouldFail() { + // Trailing comma after last pair + String ppl = "source=EMP | replace \"CLERK\" WITH \"EMPLOYEE\", IN JOB"; + getRelNode(ppl); + } + + @Test + public void testWildcardReplace_prefixWildcard() { + // Replace suffix wildcard - e.g., "*MAN" matches "SALESMAN" → "SELLER" + // Wildcard pattern is converted to regex at planning time + String ppl = "source=EMP | replace \"*MAN\" WITH \"SELLER\" IN JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[REGEXP_REPLACE($2," + + " '^\\Q\\E(.*?)\\QMAN\\E$':VARCHAR, 'SELLER':VARCHAR)], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + + verifyLogical(root, expectedLogical); + } + + @Test + public void testWildcardReplace_multipleWildcards() { + // Replace with multiple wildcards for capture and substitution + // Wildcard pattern "*_*" is converted to regex replacement "$1_$2" + String ppl = "source=EMP | replace \"* - *\" WITH \"*_*\" IN JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[REGEXP_REPLACE($2, '^\\Q\\E(.*?)\\Q -" + + " \\E(.*?)\\Q\\E$':VARCHAR, '$1_$2':VARCHAR)], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + + verifyLogical(root, expectedLogical); + } + + @Test(expected = IllegalArgumentException.class) + public void testWildcardReplace_symmetryMismatch_shouldFail() { + // Pattern has 2 wildcards, replacement has 1 - should throw error + String ppl = "source=EMP | replace \"* - *\" WITH \"*\" IN JOB"; + getRelNode(ppl); + } + + @Test + public void testWildcardReplace_symmetryValid_zeroInReplacement() { + // Pattern has 2 wildcards, replacement has 0 - should work + // Literal replacement "FIXED" has no wildcards, which is valid + String ppl = "source=EMP | replace \"* - *\" WITH \"FIXED\" IN JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[REGEXP_REPLACE($2, '^\\Q\\E(.*?)\\Q -" + + " \\E(.*?)\\Q\\E$':VARCHAR, 'FIXED':VARCHAR)], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + + verifyLogical(root, expectedLogical); + } + + @Test + public void testWildcardAndLiteralReplace_mixedPairs() { + // Multiple pairs: one with wildcard (converted to REGEXP_REPLACE), one literal (REPLACE) + String ppl = + "source=EMP | replace \"*CLERK\" WITH \"EMPLOYEE\", \"MANAGER\" WITH \"SUPERVISOR\" IN JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[REPLACE(REGEXP_REPLACE($2," + + " '^\\Q\\E(.*?)\\QCLERK\\E$':VARCHAR, 'EMPLOYEE':VARCHAR), 'MANAGER':VARCHAR," + + " 'SUPERVISOR':VARCHAR)], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6]," + + " DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + + verifyLogical(root, expectedLogical); + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRexTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRexTest.java index c73a57eadfd..4ec76823bfe 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRexTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRexTest.java @@ -28,12 +28,12 @@ public void testRexBasicFieldExtraction() { String ppl = "source=EMP | rex field=ENAME '(?[A-Z]).*' | fields ENAME, first"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(ENAME=[$1], first=[REX_EXTRACT($1, '(?[A-Z]).*', 1)])\n" + "LogicalProject(ENAME=[$1], first=[REX_EXTRACT($1, '(?[A-Z]).*', 'first')])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `ENAME`, `REX_EXTRACT`(`ENAME`, '(?[A-Z]).*', 1) `first`\n" + "SELECT `ENAME`, `REX_EXTRACT`(`ENAME`, '(?[A-Z]).*', 'first') `first`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -44,14 +44,14 @@ public void testRexMultipleNamedGroups() { "source=EMP | rex field=ENAME '(?[A-Z])(?.*)' | fields ENAME, first, rest"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(ENAME=[$1], first=[REX_EXTRACT($1, '(?[A-Z])(?.*)', 1)]," - + " rest=[REX_EXTRACT($1, '(?[A-Z])(?.*)', 2)])\n" + "LogicalProject(ENAME=[$1], first=[REX_EXTRACT($1, '(?[A-Z])(?.*)', 'first')]," + + " rest=[REX_EXTRACT($1, '(?[A-Z])(?.*)', 'rest')])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `ENAME`, `REX_EXTRACT`(`ENAME`, '(?[A-Z])(?.*)', 1) `first`," - + " `REX_EXTRACT`(`ENAME`, '(?[A-Z])(?.*)', 2) `rest`\n" + "SELECT `ENAME`, `REX_EXTRACT`(`ENAME`, '(?[A-Z])(?.*)', 'first') `first`," + + " `REX_EXTRACT`(`ENAME`, '(?[A-Z])(?.*)', 'rest') `rest`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -62,12 +62,13 @@ public void testRexWithMaxMatch() { "source=EMP | rex field=ENAME '(?[A-Z])' max_match=3 | fields ENAME, letter"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(ENAME=[$1], letter=[REX_EXTRACT_MULTI($1, '(?[A-Z])', 1, 3)])\n" + "LogicalProject(ENAME=[$1], letter=[REX_EXTRACT_MULTI($1, '(?[A-Z])', 'letter'," + + " 3)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `ENAME`, `REX_EXTRACT_MULTI`(`ENAME`, '(?[A-Z])', 1, 3) `letter`\n" + "SELECT `ENAME`, `REX_EXTRACT_MULTI`(`ENAME`, '(?[A-Z])', 'letter', 3) `letter`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -79,14 +80,14 @@ public void testRexChainedCommands() { + " fields ENAME, JOB, firstinitial, jobtype"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(ENAME=[$1], JOB=[$2], firstinitial=[REX_EXTRACT($1," - + " '(?^.)', 1)], jobtype=[REX_EXTRACT($2, '(?\\w+)', 1)])\n" + "LogicalProject(ENAME=[$1], JOB=[$2], firstinitial=[REX_EXTRACT($1, '(?^.)'," + + " 'firstinitial')], jobtype=[REX_EXTRACT($2, '(?\\w+)', 'jobtype')])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `ENAME`, `JOB`, `REX_EXTRACT`(`ENAME`, '(?^.)', 1) `firstinitial`," - + " `REX_EXTRACT`(`JOB`, '(?\\w+)', 1) `jobtype`\n" + "SELECT `ENAME`, `JOB`, `REX_EXTRACT`(`ENAME`, '(?^.)', 'firstinitial')" + + " `firstinitial`, `REX_EXTRACT`(`JOB`, '(?\\w+)', 'jobtype') `jobtype`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -98,13 +99,14 @@ public void testRexWithWhereClause() { + " SAL"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(ENAME=[$1], first=[REX_EXTRACT($1, '(?[A-Z]).*', 1)], SAL=[$5])\n" + "LogicalProject(ENAME=[$1], first=[REX_EXTRACT($1, '(?[A-Z]).*', 'first')]," + + " SAL=[$5])\n" + " LogicalFilter(condition=[>($5, 1000)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `ENAME`, `REX_EXTRACT`(`ENAME`, '(?[A-Z]).*', 1) `first`, `SAL`\n" + "SELECT `ENAME`, `REX_EXTRACT`(`ENAME`, '(?[A-Z]).*', 'first') `first`, `SAL`\n" + "FROM `scott`.`EMP`\n" + "WHERE `SAL` > 1000"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -117,14 +119,14 @@ public void testRexWithAggregation() { String expectedLogical = "LogicalProject(count()=[$1], jobtype=[$0])\n" + " LogicalAggregate(group=[{0}], count()=[COUNT()])\n" - + " LogicalProject(jobtype=[REX_EXTRACT($2, '(?\\w+)', 1)])\n" + + " LogicalProject(jobtype=[REX_EXTRACT($2, '(?\\w+)', 'jobtype')])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT COUNT(*) `count()`, `REX_EXTRACT`(`JOB`, '(?\\w+)', 1) `jobtype`\n" + "SELECT COUNT(*) `count()`, `REX_EXTRACT`(`JOB`, '(?\\w+)', 'jobtype') `jobtype`\n" + "FROM `scott`.`EMP`\n" - + "GROUP BY `REX_EXTRACT`(`JOB`, '(?\\w+)', 1)"; + + "GROUP BY `REX_EXTRACT`(`JOB`, '(?\\w+)', 'jobtype')"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -136,13 +138,14 @@ public void testRexComplexPattern() { RelNode root = getRelNode(ppl); String expectedLogical = "LogicalProject(ENAME=[$1], prefix=[REX_EXTRACT($1, '(?[A-Z]{2})(?[A-Z]+)'," - + " 1)], suffix=[REX_EXTRACT($1, '(?[A-Z]{2})(?[A-Z]+)', 2)])\n" + + " 'prefix')], suffix=[REX_EXTRACT($1, '(?[A-Z]{2})(?[A-Z]+)'," + + " 'suffix')])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `ENAME`, `REX_EXTRACT`(`ENAME`, '(?[A-Z]{2})(?[A-Z]+)', 1)" - + " `prefix`, `REX_EXTRACT`(`ENAME`, '(?[A-Z]{2})(?[A-Z]+)', 2)" + "SELECT `ENAME`, `REX_EXTRACT`(`ENAME`, '(?[A-Z]{2})(?[A-Z]+)', 'prefix')" + + " `prefix`, `REX_EXTRACT`(`ENAME`, '(?[A-Z]{2})(?[A-Z]+)', 'suffix')" + " `suffix`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -157,12 +160,13 @@ public void testRexWithSort() { String expectedLogical = "LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], fetch=[5])\n" + " LogicalProject(ENAME=[$1], firstletter=[REX_EXTRACT($1, '(?^.)'," - + " 1)])\n" + + " 'firstletter')])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `ENAME`, `REX_EXTRACT`(`ENAME`, '(?^.)', 1) `firstletter`\n" + "SELECT `ENAME`, `REX_EXTRACT`(`ENAME`, '(?^.)', 'firstletter')" + + " `firstletter`\n" + "FROM `scott`.`EMP`\n" + "ORDER BY 2\n" + "LIMIT 5"; @@ -176,12 +180,13 @@ public void testRexWithMaxMatchZero() { "source=EMP | rex field=ENAME '(?[A-Z])' max_match=0 | fields ENAME, letter"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(ENAME=[$1], letter=[REX_EXTRACT_MULTI($1, '(?[A-Z])', 1, 10)])\n" + "LogicalProject(ENAME=[$1], letter=[REX_EXTRACT_MULTI($1, '(?[A-Z])', 'letter'," + + " 10)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `ENAME`, `REX_EXTRACT_MULTI`(`ENAME`, '(?[A-Z])', 1, 10) `letter`\n" + "SELECT `ENAME`, `REX_EXTRACT_MULTI`(`ENAME`, '(?[A-Z])', 'letter', 10) `letter`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -200,12 +205,13 @@ public void testRexWithMaxMatchWithinLimit() { "source=EMP | rex field=ENAME '(?[A-Z])' max_match=5 | fields ENAME, letter"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(ENAME=[$1], letter=[REX_EXTRACT_MULTI($1, '(?[A-Z])', 1, 5)])\n" + "LogicalProject(ENAME=[$1], letter=[REX_EXTRACT_MULTI($1, '(?[A-Z])', 'letter'," + + " 5)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `ENAME`, `REX_EXTRACT_MULTI`(`ENAME`, '(?[A-Z])', 1, 5) `letter`\n" + "SELECT `ENAME`, `REX_EXTRACT_MULTI`(`ENAME`, '(?[A-Z])', 'letter', 5) `letter`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -217,12 +223,13 @@ public void testRexWithMaxMatchAtLimit() { "source=EMP | rex field=ENAME '(?[A-Z])' max_match=10 | fields ENAME, letter"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(ENAME=[$1], letter=[REX_EXTRACT_MULTI($1, '(?[A-Z])', 1, 10)])\n" + "LogicalProject(ENAME=[$1], letter=[REX_EXTRACT_MULTI($1, '(?[A-Z])', 'letter'," + + " 10)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `ENAME`, `REX_EXTRACT_MULTI`(`ENAME`, '(?[A-Z])', 1, 10) `letter`\n" + "SELECT `ENAME`, `REX_EXTRACT_MULTI`(`ENAME`, '(?[A-Z])', 'letter', 10) `letter`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -248,13 +255,13 @@ public void testRexWithOffsetField() { + " first, offsets"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(ENAME=[$1], first=[REX_EXTRACT($1, '(?[A-Z]).*', 1)]," + "LogicalProject(ENAME=[$1], first=[REX_EXTRACT($1, '(?[A-Z]).*', 'first')]," + " offsets=[REX_OFFSET($1, '(?[A-Z]).*')])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `ENAME`, `REX_EXTRACT`(`ENAME`, '(?[A-Z]).*', 1) `first`," + "SELECT `ENAME`, `REX_EXTRACT`(`ENAME`, '(?[A-Z]).*', 'first') `first`," + " `REX_OFFSET`(`ENAME`, '(?[A-Z]).*') `offsets`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -267,15 +274,15 @@ public void testRexWithMultipleNamedGroupsAndOffsetField() { + " ENAME, first, rest, positions"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(ENAME=[$1], first=[REX_EXTRACT($1, '(?[A-Z])(?.*)', 1)]," - + " rest=[REX_EXTRACT($1, '(?[A-Z])(?.*)', 2)], positions=[REX_OFFSET($1," - + " '(?[A-Z])(?.*)')])\n" + "LogicalProject(ENAME=[$1], first=[REX_EXTRACT($1, '(?[A-Z])(?.*)', 'first')]," + + " rest=[REX_EXTRACT($1, '(?[A-Z])(?.*)', 'rest')]," + + " positions=[REX_OFFSET($1, '(?[A-Z])(?.*)')])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `ENAME`, `REX_EXTRACT`(`ENAME`, '(?[A-Z])(?.*)', 1) `first`," - + " `REX_EXTRACT`(`ENAME`, '(?[A-Z])(?.*)', 2) `rest`," + "SELECT `ENAME`, `REX_EXTRACT`(`ENAME`, '(?[A-Z])(?.*)', 'first') `first`," + + " `REX_EXTRACT`(`ENAME`, '(?[A-Z])(?.*)', 'rest') `rest`," + " `REX_OFFSET`(`ENAME`, '(?[A-Z])(?.*)') `positions`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -288,13 +295,13 @@ public void testRexWithMaxMatchAndOffsetField() { + " fields ENAME, letter, positions"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(ENAME=[$1], letter=[REX_EXTRACT_MULTI($1, '(?[A-Z])', 1, 3)]," - + " positions=[REX_OFFSET($1, '(?[A-Z])')])\n" + "LogicalProject(ENAME=[$1], letter=[REX_EXTRACT_MULTI($1, '(?[A-Z])', 'letter'," + + " 3)], positions=[REX_OFFSET($1, '(?[A-Z])')])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `ENAME`, `REX_EXTRACT_MULTI`(`ENAME`, '(?[A-Z])', 1, 3) `letter`," + "SELECT `ENAME`, `REX_EXTRACT_MULTI`(`ENAME`, '(?[A-Z])', 'letter', 3) `letter`," + " `REX_OFFSET`(`ENAME`, '(?[A-Z])') `positions`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLScalarSubqueryTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLScalarSubqueryTest.java index a0029e21d64..e19d896283d 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLScalarSubqueryTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLScalarSubqueryTest.java @@ -269,11 +269,13 @@ public void testTwoScalarSubqueriesInOr() { "SELECT *\n" + "FROM `scott`.`EMP`\n" + "WHERE `SAL` = (((SELECT MAX(`HISAL`) `max(HISAL)`\n" + + "FROM (SELECT `HISAL`\n" + "FROM `scott`.`SALGRADE`\n" - + "ORDER BY `LOSAL`))) OR `SAL` = (((SELECT MIN(`HISAL`) `min(HISAL)`\n" + + "ORDER BY `LOSAL`) `t0`))) OR `SAL` = (((SELECT MIN(`HISAL`) `min(HISAL)`\n" + + "FROM (SELECT `HISAL`\n" + "FROM `scott`.`SALGRADE`\n" + "WHERE `LOSAL` > 1000.0\n" - + "ORDER BY `HISAL` DESC)))"; + + "ORDER BY `HISAL` DESC) `t4`)))"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -339,11 +341,34 @@ public void testNestedScalarSubquery() { verifyPPLToSparkSQL(root, expectedSparkSql); } - // TODO: With Calcite, we can add more complex scalar subquery, such as - // stats by a scalar subquery: - // | eval count_a = [ - // source=.. - // ] - // | stats .. by count_a - // But currently, statsBy an expression is unsupported in PPL. + @Test + public void testCorrelatedScalarSubqueryInWhereMaxOut() { + String ppl = + """ + source=EMP + | where SAL > [ + source=SALGRADE | where SAL = HISAL | stats AVG(SAL) + ] + """; + RelNode root = getRelNode(ppl); + String expectedLogical = + "" + + "LogicalFilter(condition=[>($5, $SCALAR_QUERY({\n" + + "LogicalAggregate(group=[{}], AVG(SAL)=[AVG($0)])\n" + + " LogicalProject($f0=[$cor0.SAL])\n" + + " LogicalFilter(condition=[=($cor0.SAL, $2)])\n" + + " LogicalTableScan(table=[[scott, SALGRADE]])\n" + + "}))], variablesSet=[[$cor0]])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "" + + "SELECT *\n" + + "FROM `scott`.`EMP`\n" + + "WHERE `SAL` > (((SELECT AVG(`EMP`.`SAL`) `AVG(SAL)`\n" + + "FROM `scott`.`SALGRADE`\n" + + "WHERE `EMP`.`SAL` = `HISAL`)))"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStreamstatsTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStreamstatsTest.java new file mode 100644 index 00000000000..04f4c7610d9 --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStreamstatsTest.java @@ -0,0 +1,189 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.calcite; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.test.CalciteAssert; +import org.junit.Test; + +public class CalcitePPLStreamstatsTest extends CalcitePPLAbstractTest { + + public CalcitePPLStreamstatsTest() { + super(CalciteAssert.SchemaSpec.SCOTT_WITH_TEMPORAL); + } + + @Test + public void testStreamstatsBy() { + String ppl = "source=EMP | streamstats max(SAL) by DEPTNO"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], max(SAL)=[$9])\n" + + " LogicalSort(sort0=[$8], dir0=[ASC])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __stream_seq__=[$8], max(SAL)=[MAX($5) OVER" + + " (PARTITION BY $7 ROWS UNBOUNDED PRECEDING)])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __stream_seq__=[ROW_NUMBER() OVER ()])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, MAX(`SAL`)" + + " OVER (PARTITION BY `DEPTNO` ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)" + + " `max(SAL)`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " ROW_NUMBER() OVER () `__stream_seq__`\n" + + "FROM `scott`.`EMP`) `t`\n" + + "ORDER BY `__stream_seq__` NULLS LAST"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testStreamstatsCurrent() { + String ppl = "source=EMP | streamstats current = false max(SAL)"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], max(SAL)=[MAX($5) OVER (ROWS BETWEEN UNBOUNDED PRECEDING" + + " AND 1 PRECEDING)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, MAX(`SAL`)" + + " OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) `max(SAL)`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testStreamstatsWindow() { + String ppl = "source=EMP | streamstats window = 5 max(SAL) by DEPTNO"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], max(SAL)=[$9])\n" + + " LogicalSort(sort0=[$8], dir0=[ASC])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{7," + + " 8}])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __stream_seq__=[ROW_NUMBER() OVER ()])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{}], max(SAL)=[MAX($5)])\n" + + " LogicalFilter(condition=[AND(>=($8, -($cor0.__stream_seq__, 4)), <=($8," + + " $cor0.__stream_seq__), =($7, $cor0.DEPTNO))])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __stream_seq__=[ROW_NUMBER() OVER ()])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `$cor0`.`EMPNO`, `$cor0`.`ENAME`, `$cor0`.`JOB`, `$cor0`.`MGR`, `$cor0`.`HIREDATE`," + + " `$cor0`.`SAL`, `$cor0`.`COMM`, `$cor0`.`DEPTNO`, `t2`.`max(SAL)`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " ROW_NUMBER() OVER () `__stream_seq__`\n" + + "FROM `scott`.`EMP`) `$cor0`,\n" + + "LATERAL (SELECT MAX(`SAL`) `max(SAL)`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " ROW_NUMBER() OVER () `__stream_seq__`\n" + + "FROM `scott`.`EMP`) `t0`\n" + + "WHERE `__stream_seq__` >= `$cor0`.`__stream_seq__` - 4 AND `__stream_seq__` <=" + + " `$cor0`.`__stream_seq__` AND `DEPTNO` = `$cor0`.`DEPTNO`) `t2`\n" + + "ORDER BY `$cor0`.`__stream_seq__` NULLS LAST"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testStreamstatsGlobal() { + String ppl = "source=EMP | streamstats window = 5 global= false max(SAL) by DEPTNO"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], max(SAL)=[$9])\n" + + " LogicalSort(sort0=[$8], dir0=[ASC])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __stream_seq__=[$8], max(SAL)=[MAX($5) OVER" + + " (PARTITION BY $7 ROWS 4 PRECEDING)])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __stream_seq__=[ROW_NUMBER() OVER ()])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, MAX(`SAL`)" + + " OVER (PARTITION BY `DEPTNO` ROWS BETWEEN 4 PRECEDING AND CURRENT ROW) `max(SAL)`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " ROW_NUMBER() OVER () `__stream_seq__`\n" + + "FROM `scott`.`EMP`) `t`\n" + + "ORDER BY `__stream_seq__` NULLS LAST"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testStreamstatsReset() { + String ppl = + "source=EMP | streamstats reset_before=SAL>100 reset_after=SAL<50 avg(SAL) by DEPTNO"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], avg(SAL)=[$12])\n" + + " LogicalSort(sort0=[$8], dir0=[ASC])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{7, 8," + + " 11}])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __stream_seq__=[$8], __reset_before_flag__=[$9]," + + " __reset_after_flag__=[$10], __seg_id__=[+(SUM($9) OVER (ROWS UNBOUNDED PRECEDING)," + + " COALESCE(SUM($10) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0))])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __stream_seq__=[ROW_NUMBER() OVER ()]," + + " __reset_before_flag__=[CASE(>($5, 100), 1, 0)], __reset_after_flag__=[CASE(<($5," + + " 50), 1, 0)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{}], avg(SAL)=[AVG($5)])\n" + + " LogicalFilter(condition=[AND(<=($8, $cor0.__stream_seq__), =($11," + + " $cor0.__seg_id__), =($7, $cor0.DEPTNO))])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __stream_seq__=[$8], __reset_before_flag__=[$9]," + + " __reset_after_flag__=[$10], __seg_id__=[+(SUM($9) OVER (ROWS UNBOUNDED PRECEDING)," + + " COALESCE(SUM($10) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0))])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3]," + + " HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], __stream_seq__=[ROW_NUMBER() OVER" + + " ()], __reset_before_flag__=[CASE(>($5, 100), 1, 0)]," + + " __reset_after_flag__=[CASE(<($5, 50), 1, 0)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `$cor0`.`EMPNO`, `$cor0`.`ENAME`, `$cor0`.`JOB`, `$cor0`.`MGR`, `$cor0`.`HIREDATE`," + + " `$cor0`.`SAL`, `$cor0`.`COMM`, `$cor0`.`DEPTNO`, `t4`.`avg(SAL)`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " `__stream_seq__`, `__reset_before_flag__`, `__reset_after_flag__`," + + " (SUM(`__reset_before_flag__`) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT" + + " ROW)) + COALESCE(SUM(`__reset_after_flag__`) OVER (ROWS BETWEEN UNBOUNDED PRECEDING" + + " AND 1 PRECEDING), 0) `__seg_id__`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " ROW_NUMBER() OVER () `__stream_seq__`, CASE WHEN `SAL` > 100 THEN 1 ELSE 0 END" + + " `__reset_before_flag__`, CASE WHEN `SAL` < 50 THEN 1 ELSE 0 END" + + " `__reset_after_flag__`\n" + + "FROM `scott`.`EMP`) `t`) `$cor0`,\n" + + "LATERAL (SELECT AVG(`SAL`) `avg(SAL)`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " `__stream_seq__`, `__reset_before_flag__`, `__reset_after_flag__`," + + " (SUM(`__reset_before_flag__`) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT" + + " ROW)) + COALESCE(SUM(`__reset_after_flag__`) OVER (ROWS BETWEEN UNBOUNDED PRECEDING" + + " AND 1 PRECEDING), 0) `__seg_id__`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " ROW_NUMBER() OVER () `__stream_seq__`, CASE WHEN `SAL` > 100 THEN 1 ELSE 0 END" + + " `__reset_before_flag__`, CASE WHEN `SAL` < 50 THEN 1 ELSE 0 END" + + " `__reset_after_flag__`\n" + + "FROM `scott`.`EMP`) `t1`) `t2`\n" + + "WHERE `__stream_seq__` <= `$cor0`.`__stream_seq__` AND `__seg_id__` =" + + " `$cor0`.`__seg_id__` AND `DEPTNO` = `$cor0`.`DEPTNO`) `t4`\n" + + "ORDER BY `$cor0`.`__stream_seq__` NULLS LAST"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java index d52d915d507..6f02d4f685a 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java @@ -53,7 +53,7 @@ public void testLike() { String expectedLogical = "" + "LogicalAggregate(group=[{}], cnt=[COUNT()])\n" - + " LogicalFilter(condition=[ILIKE($2, 'SALE%':VARCHAR, '\\')])\n" + + " LogicalFilter(condition=[ILIKE($2, 'SALE%', '\\')])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedResult = "cnt=4\n"; @@ -263,4 +263,53 @@ public void testRegexMatchWithStats() { + "WHERE REGEXP_CONTAINS(`JOB`, 'MAN')"; verifyPPLToSparkSQL(root, expectedSparkSql); } + + @Test + public void testReplaceLiteralString() { + // Test basic literal string replacement - replaces all 'A' with 'X' + String ppl = "source=EMP | eval new_name = replace(ENAME, 'A', 'X') | fields ENAME, new_name"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(ENAME=[$1], new_name=[REGEXP_REPLACE($1, 'A', 'X')])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `ENAME`, REGEXP_REPLACE(`ENAME`, 'A', 'X') `new_name`\n" + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testReplaceWithRegexPattern() { + // Test regex pattern - remove all digits + String ppl = "source=EMP | eval no_digits = replace(JOB, '\\\\d+', '') | fields JOB, no_digits"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(JOB=[$2], no_digits=[REGEXP_REPLACE($2, '\\d+':VARCHAR, '':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `JOB`, REGEXP_REPLACE(`JOB`, '\\d+', '') `no_digits`\n" + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testReplaceWithRegexCaptureGroups() { + // Test regex with capture groups - swap first two characters using \1 and \2 backreferences + String ppl = + "source=EMP | eval swapped = replace(ENAME, '^(.)(.)', '\\\\2\\\\1') | fields ENAME," + + " swapped"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(ENAME=[$1], swapped=[REGEXP_REPLACE($1, '^(.)(.)':VARCHAR," + + " '$2$1')])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `ENAME`, REGEXP_REPLACE(`ENAME`, '^(.)(.)', '$2$1') `swapped`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimechartTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimechartTest.java index 703850ccfff..ee6b82f2d85 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimechartTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimechartTest.java @@ -6,6 +6,7 @@ package org.opensearch.sql.ppl.calcite; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; import com.google.common.collect.ImmutableList; import java.util.List; @@ -81,6 +82,62 @@ public void testTimechartBasic() { verifyPPLToSparkSQL(root, expectedSparkSql); } + @Test + public void testTimechartPerSecond() { + withPPLQuery("source=events | timechart per_second(cpu_usage)") + .expectSparkSQL( + "SELECT `@timestamp`, `DIVIDE`(`per_second(cpu_usage)` * 1.0000E3," + + " TIMESTAMPDIFF('MILLISECOND', `@timestamp`, TIMESTAMPADD('MINUTE', 1," + + " `@timestamp`))) `per_second(cpu_usage)`\n" + + "FROM (SELECT `SPAN`(`@timestamp`, 1, 'm') `@timestamp`, SUM(`cpu_usage`)" + + " `per_second(cpu_usage)`\n" + + "FROM `scott`.`events`\n" + + "GROUP BY `SPAN`(`@timestamp`, 1, 'm')\n" + + "ORDER BY 1 NULLS LAST) `t2`"); + } + + @Test + public void testTimechartPerMinute() { + withPPLQuery("source=events | timechart per_minute(cpu_usage)") + .expectSparkSQL( + "SELECT `@timestamp`, `DIVIDE`(`per_minute(cpu_usage)` * 6.00000E4," + + " TIMESTAMPDIFF('MILLISECOND', `@timestamp`, TIMESTAMPADD('MINUTE', 1," + + " `@timestamp`))) `per_minute(cpu_usage)`\n" + + "FROM (SELECT `SPAN`(`@timestamp`, 1, 'm') `@timestamp`, SUM(`cpu_usage`)" + + " `per_minute(cpu_usage)`\n" + + "FROM `scott`.`events`\n" + + "GROUP BY `SPAN`(`@timestamp`, 1, 'm')\n" + + "ORDER BY 1 NULLS LAST) `t2`"); + } + + @Test + public void testTimechartPerHour() { + withPPLQuery("source=events | timechart per_hour(cpu_usage)") + .expectSparkSQL( + "SELECT `@timestamp`, `DIVIDE`(`per_hour(cpu_usage)` * 3.6000000E6," + + " TIMESTAMPDIFF('MILLISECOND', `@timestamp`, TIMESTAMPADD('MINUTE', 1," + + " `@timestamp`))) `per_hour(cpu_usage)`\n" + + "FROM (SELECT `SPAN`(`@timestamp`, 1, 'm') `@timestamp`, SUM(`cpu_usage`)" + + " `per_hour(cpu_usage)`\n" + + "FROM `scott`.`events`\n" + + "GROUP BY `SPAN`(`@timestamp`, 1, 'm')\n" + + "ORDER BY 1 NULLS LAST) `t2`"); + } + + @Test + public void testTimechartPerDay() { + withPPLQuery("source=events | timechart per_day(cpu_usage)") + .expectSparkSQL( + "SELECT `@timestamp`, `DIVIDE`(`per_day(cpu_usage)` * 8.64E7," + + " TIMESTAMPDIFF('MILLISECOND', `@timestamp`, TIMESTAMPADD('MINUTE', 1," + + " `@timestamp`))) `per_day(cpu_usage)`\n" + + "FROM (SELECT `SPAN`(`@timestamp`, 1, 'm') `@timestamp`, SUM(`cpu_usage`)" + + " `per_day(cpu_usage)`\n" + + "FROM `scott`.`events`\n" + + "GROUP BY `SPAN`(`@timestamp`, 1, 'm')\n" + + "ORDER BY 1 NULLS LAST) `t2`"); + } + @Test public void testTimechartWithSpan() { String ppl = "source=events | timechart span=1h count()"; @@ -328,6 +385,13 @@ public void testTimechartWithUseOtherBeforeLimit() { assertNotNull(plan); } + @Test + public void testTimechartUsingZeroSpanShouldThrow() { + String ppl = "source=events | timechart span=0h limit=5 count() by host"; + Throwable t = assertThrows(IllegalArgumentException.class, () -> parsePPL(ppl)); + verifyErrorMessageContains(t, "Zero or negative time interval not supported: 0h"); + } + private UnresolvedPlan parsePPL(String query) { PPLSyntaxParser parser = new PPLSyntaxParser(); AstBuilder astBuilder = new AstBuilder(query); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTrendlineTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTrendlineTest.java index 582077b82b2..b036a4b5906 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTrendlineTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTrendlineTest.java @@ -44,19 +44,19 @@ public void testTrendlineWma() { String expectedLogical = "LogicalProject(SAL=[$5], SAL_trendline=[CASE(>(COUNT() OVER (ROWS 2 PRECEDING), 2)," - + " /(+(+(CAST(CAST(NTH_VALUE($5, 1) OVER (ROWS 2 PRECEDING)):DECIMAL(17," - + " 2)):DECIMAL(18, 2), *(NTH_VALUE($5, 2) OVER (ROWS 2 PRECEDING), 2))," - + " *(NTH_VALUE($5, 3) OVER (ROWS 2 PRECEDING), 3)), 6.0E0:DOUBLE), null:NULL)])\n" + + " /(+(+(CAST(NTH_VALUE($5, 1) OVER (ROWS 2 PRECEDING)):DECIMAL(18, 2)," + + " *(NTH_VALUE($5, 2) OVER (ROWS 2 PRECEDING), 2)), *(NTH_VALUE($5, 3) OVER (ROWS 2" + + " PRECEDING), 3)), 6.0E0:DOUBLE), null:NULL)])\n" + " LogicalFilter(condition=[IS NOT NULL($5)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = "SELECT `SAL`, CASE WHEN (COUNT(*) OVER (ROWS BETWEEN 2 PRECEDING AND CURRENT ROW)) > 2" - + " THEN (CAST(CAST(NTH_VALUE(`SAL`, 1) OVER (ROWS BETWEEN 2 PRECEDING AND CURRENT ROW)" - + " AS DECIMAL(17, 2)) AS DECIMAL(18, 2)) + (NTH_VALUE(`SAL`, 2) OVER (ROWS BETWEEN 2" - + " PRECEDING AND CURRENT ROW)) * 2 + (NTH_VALUE(`SAL`, 3) OVER (ROWS BETWEEN 2" - + " PRECEDING AND CURRENT ROW)) * 3) / 6.0E0 ELSE NULL END `SAL_trendline`\n" + + " THEN (CAST(NTH_VALUE(`SAL`, 1) OVER (ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS" + + " DECIMAL(18, 2)) + (NTH_VALUE(`SAL`, 2) OVER (ROWS BETWEEN 2 PRECEDING AND CURRENT" + + " ROW)) * 2 + (NTH_VALUE(`SAL`, 3) OVER (ROWS BETWEEN 2 PRECEDING AND CURRENT ROW)) *" + + " 3) / 6.0E0 ELSE NULL END `SAL_trendline`\n" + "FROM `scott`.`EMP`\n" + "WHERE `SAL` IS NOT NULL"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -71,8 +71,8 @@ public void testTrendlineMultipleFields() { String expectedLogical = "LogicalProject(SAL_trendline=[CASE(>(COUNT() OVER (ROWS 1 PRECEDING), 1)," - + " /(+(CAST(CAST(NTH_VALUE($5, 1) OVER (ROWS 1 PRECEDING)):DECIMAL(17, 2)):DECIMAL(18," - + " 2), *(NTH_VALUE($5, 2) OVER (ROWS 1 PRECEDING), 2)), 3.0E0:DOUBLE), null:NULL)]," + + " /(+(CAST(NTH_VALUE($5, 1) OVER (ROWS 1 PRECEDING)):DECIMAL(18, 2), *(NTH_VALUE($5," + + " 2) OVER (ROWS 1 PRECEDING), 2)), 3.0E0:DOUBLE), null:NULL)]," + " DEPTNO_trendline=[CASE(>(COUNT() OVER (ROWS 1 PRECEDING), 1), /(SUM($7) OVER (ROWS" + " 1 PRECEDING), CAST(COUNT($7) OVER (ROWS 1 PRECEDING)):DOUBLE NOT NULL)," + " null:NULL)])\n" @@ -83,12 +83,12 @@ public void testTrendlineMultipleFields() { String expectedSparkSql = "SELECT CASE WHEN (COUNT(*) OVER (ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)) > 1 THEN" - + " (CAST(CAST(NTH_VALUE(`SAL`, 1) OVER (ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS" - + " DECIMAL(17, 2)) AS DECIMAL(18, 2)) + (NTH_VALUE(`SAL`, 2) OVER (ROWS BETWEEN 1" - + " PRECEDING AND CURRENT ROW)) * 2) / 3.0E0 ELSE NULL END `SAL_trendline`, CASE WHEN" - + " (COUNT(*) OVER (ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)) > 1 THEN (SUM(`DEPTNO`)" - + " OVER (ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)) / CAST(COUNT(`DEPTNO`) OVER (ROWS" - + " BETWEEN 1 PRECEDING AND CURRENT ROW) AS DOUBLE) ELSE NULL END `DEPTNO_trendline`\n" + + " (CAST(NTH_VALUE(`SAL`, 1) OVER (ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS" + + " DECIMAL(18, 2)) + (NTH_VALUE(`SAL`, 2) OVER (ROWS BETWEEN 1 PRECEDING AND CURRENT" + + " ROW)) * 2) / 3.0E0 ELSE NULL END `SAL_trendline`, CASE WHEN (COUNT(*) OVER (ROWS" + + " BETWEEN 1 PRECEDING AND CURRENT ROW)) > 1 THEN (SUM(`DEPTNO`) OVER (ROWS BETWEEN 1" + + " PRECEDING AND CURRENT ROW)) / CAST(COUNT(`DEPTNO`) OVER (ROWS BETWEEN 1 PRECEDING" + + " AND CURRENT ROW) AS DOUBLE) ELSE NULL END `DEPTNO_trendline`\n" + "FROM (SELECT *\n" + "FROM `scott`.`EMP`\n" + "WHERE `SAL` IS NOT NULL) `t`\n" diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index 99614657b1e..a1823e4befe 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -8,6 +8,7 @@ import static java.util.Collections.emptyList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.when; import static org.opensearch.sql.ast.dsl.AstDSL.agg; import static org.opensearch.sql.ast.dsl.AstDSL.aggregate; @@ -22,6 +23,7 @@ import static org.opensearch.sql.ast.dsl.AstDSL.defaultSortFieldArgs; import static org.opensearch.sql.ast.dsl.AstDSL.defaultStatsArgs; import static org.opensearch.sql.ast.dsl.AstDSL.describe; +import static org.opensearch.sql.ast.dsl.AstDSL.doubleLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.eval; import static org.opensearch.sql.ast.dsl.AstDSL.exprList; import static org.opensearch.sql.ast.dsl.AstDSL.field; @@ -75,9 +77,11 @@ import org.opensearch.sql.ast.tree.Kmeans; import org.opensearch.sql.ast.tree.ML; import org.opensearch.sql.ast.tree.RareTopN.CommandType; +import org.opensearch.sql.ast.tree.Timechart; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.setting.Settings.Key; +import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; import org.opensearch.sql.utils.SystemIndexUtils; @@ -518,9 +522,9 @@ public void testSortCommandWithD() { } @Test - public void testSortCommandWithMultipleFieldsAndDesc() { + public void testSortCommandWithMixedSuffixSyntax() { assertEqual( - "source=t | sort f1, -f2 desc", + "source=t | sort f1 desc, f2 asc", sort( relation("t"), field( @@ -554,9 +558,9 @@ public void testSortCommandWithA() { } @Test - public void testSortCommandWithMultipleFieldsAndAsc() { + public void testSortCommandWithMixedPrefixSyntax() { assertEqual( - "source=t | sort f1, f2 asc", + "source=t | sort +f1, -f2", sort( relation("t"), field( @@ -564,6 +568,95 @@ public void testSortCommandWithMultipleFieldsAndAsc() { exprList(argument("asc", booleanLiteral(true)), argument("type", nullLiteral()))), field( "f2", + exprList( + argument("asc", booleanLiteral(false)), argument("type", nullLiteral()))))); + } + + @Test + public void testSortCommandMixedSyntaxValidation() { + assertThrows(SemanticCheckException.class, () -> plan("source=t | sort +f1, f2 desc")); + assertThrows(SemanticCheckException.class, () -> plan("source=t | sort f1 asc, +f2")); + } + + @Test + public void testSortCommandSingleFieldMixedSyntaxError() { + SemanticCheckException exception = + assertThrows(SemanticCheckException.class, () -> plan("source=t | sort -salary desc")); + + assertTrue( + exception + .getMessage() + .contains( + "Cannot use both prefix (-) and suffix (desc) sort direction syntax on the same" + + " field")); + } + + @Test + public void testSortCommandMultipleSuffixSyntax() { + assertEqual( + "source=t | sort f1 asc, f2 desc, f3 asc", + sort( + relation("t"), + field( + "f1", + exprList(argument("asc", booleanLiteral(true)), argument("type", nullLiteral()))), + field( + "f2", + exprList(argument("asc", booleanLiteral(false)), argument("type", nullLiteral()))), + field( + "f3", + exprList(argument("asc", booleanLiteral(true)), argument("type", nullLiteral()))))); + } + + @Test + public void testSortCommandMixingPrefixWithDefault() { + assertEqual( + "source=t | sort +f1, f2, -f3", + sort( + relation("t"), + field( + "f1", + exprList(argument("asc", booleanLiteral(true)), argument("type", nullLiteral()))), + field( + "f2", + exprList(argument("asc", booleanLiteral(true)), argument("type", nullLiteral()))), + field( + "f3", + exprList( + argument("asc", booleanLiteral(false)), argument("type", nullLiteral()))))); + } + + @Test + public void testSortCommandMixingSuffixWithDefault() { + assertEqual( + "source=t | sort f1, f2 desc, f3 asc", + sort( + relation("t"), + field( + "f1", + exprList(argument("asc", booleanLiteral(true)), argument("type", nullLiteral()))), + field( + "f2", + exprList(argument("asc", booleanLiteral(false)), argument("type", nullLiteral()))), + field( + "f3", + exprList(argument("asc", booleanLiteral(true)), argument("type", nullLiteral()))))); + } + + @Test + public void testSortCommandAllDefaultFields() { + assertEqual( + "source=t | sort f1, f2, f3", + sort( + relation("t"), + field( + "f1", + exprList(argument("asc", booleanLiteral(true)), argument("type", nullLiteral()))), + field( + "f2", + exprList(argument("asc", booleanLiteral(true)), argument("type", nullLiteral()))), + field( + "f3", exprList(argument("asc", booleanLiteral(true)), argument("type", nullLiteral()))))); } @@ -636,7 +729,8 @@ public void testRareCommand() { exprList( argument("noOfResults", intLiteral(10)), argument("countField", stringLiteral("count")), - argument("showCount", booleanLiteral(true))), + argument("showCount", booleanLiteral(true)), + argument("useNull", booleanLiteral(true))), emptyList(), field("a"))); } @@ -651,7 +745,8 @@ public void testRareCommandWithGroupBy() { exprList( argument("noOfResults", intLiteral(10)), argument("countField", stringLiteral("count")), - argument("showCount", booleanLiteral(true))), + argument("showCount", booleanLiteral(true)), + argument("useNull", booleanLiteral(true))), exprList(field("b")), field("a"))); } @@ -666,7 +761,8 @@ public void testRareCommandWithMultipleFields() { exprList( argument("noOfResults", intLiteral(10)), argument("countField", stringLiteral("count")), - argument("showCount", booleanLiteral(true))), + argument("showCount", booleanLiteral(true)), + argument("useNull", booleanLiteral(true))), exprList(field("c")), field("a"), field("b"))); @@ -682,7 +778,8 @@ public void testTopCommandWithN() { exprList( argument("noOfResults", intLiteral(1)), argument("countField", stringLiteral("count")), - argument("showCount", booleanLiteral(true))), + argument("showCount", booleanLiteral(true)), + argument("useNull", booleanLiteral(true))), emptyList(), field("a"))); } @@ -697,7 +794,8 @@ public void testTopCommandWithoutNAndGroupBy() { exprList( argument("noOfResults", intLiteral(10)), argument("countField", stringLiteral("count")), - argument("showCount", booleanLiteral(true))), + argument("showCount", booleanLiteral(true)), + argument("useNull", booleanLiteral(true))), emptyList(), field("a"))); } @@ -712,7 +810,8 @@ public void testTopCommandWithNAndGroupBy() { exprList( argument("noOfResults", intLiteral(1)), argument("countField", stringLiteral("count")), - argument("showCount", booleanLiteral(true))), + argument("showCount", booleanLiteral(true)), + argument("useNull", booleanLiteral(true))), exprList(field("b")), field("a"))); } @@ -727,12 +826,46 @@ public void testTopCommandWithMultipleFields() { exprList( argument("noOfResults", intLiteral(1)), argument("countField", stringLiteral("count")), - argument("showCount", booleanLiteral(true))), + argument("showCount", booleanLiteral(true)), + argument("useNull", booleanLiteral(true))), exprList(field("c")), field("a"), field("b"))); } + @Test + public void testTopCommandWithUseNullFalse() { + assertEqual( + "source=t | top 1 usenull=false a by b", + rareTopN( + relation("t"), + CommandType.TOP, + exprList( + argument("noOfResults", intLiteral(1)), + argument("countField", stringLiteral("count")), + argument("showCount", booleanLiteral(true)), + argument("useNull", booleanLiteral(false))), + exprList(field("b")), + field("a"))); + } + + @Test + public void testTopCommandWithLegacyFalse() { + when(settings.getSettingValue(Key.PPL_SYNTAX_LEGACY_PREFERRED)).thenReturn(false); + assertEqual( + "source=t | top 1 a by b", + rareTopN( + relation("t"), + CommandType.TOP, + exprList( + argument("noOfResults", intLiteral(1)), + argument("countField", stringLiteral("count")), + argument("showCount", booleanLiteral(true)), + argument("useNull", booleanLiteral(false))), + exprList(field("b")), + field("a"))); + } + @Test public void testGrokCommand() { assertEqual( @@ -855,6 +988,19 @@ public void testFillNullCommandVariousValues() { Pair.of(field("c"), intLiteral(3))))); } + @Test + public void testFillNullValueAllFields() { + assertEqual( + "source=t | fillnull value=\"N/A\"", fillNull(relation("t"), stringLiteral("N/A"), true)); + } + + @Test + public void testFillNullValueWithFields() { + assertEqual( + "source=t | fillnull value=0 a, b, c", + fillNull(relation("t"), intLiteral(0), true, field("a"), field("b"), field("c"))); + } + public void testTrendline() { assertEqual( "source=t | trendline sma(5, test_field) as test_field_alias sma(1, test_field_2) as" @@ -1076,6 +1222,126 @@ public void testPatternsWithoutArguments() { ImmutableMap.of())); } + @Test + public void testTimechartWithPerSecondFunction() { + assertEqual( + "source=t | timechart per_second(a)", + eval( + new Timechart(relation("t"), alias("per_second(a)", aggregate("sum", field("a")))) + .span(span(field("@timestamp"), intLiteral(1), SpanUnit.of("m"))) + .limit(10) + .useOther(true), + let( + field("per_second(a)"), + function( + "/", + function("*", field("per_second(a)"), doubleLiteral(1000.0)), + function( + "timestampdiff", + stringLiteral("MILLISECOND"), + field("@timestamp"), + function( + "timestampadd", + stringLiteral("MINUTE"), + intLiteral(1), + field("@timestamp"))))))); + } + + @Test + public void testTimechartWithPerMinuteFunction() { + assertEqual( + "source=t | timechart per_minute(a)", + eval( + new Timechart(relation("t"), alias("per_minute(a)", aggregate("sum", field("a")))) + .span(span(field("@timestamp"), intLiteral(1), SpanUnit.of("m"))) + .limit(10) + .useOther(true), + let( + field("per_minute(a)"), + function( + "/", + function("*", field("per_minute(a)"), doubleLiteral(60000.0)), + function( + "timestampdiff", + stringLiteral("MILLISECOND"), + field("@timestamp"), + function( + "timestampadd", + stringLiteral("MINUTE"), + intLiteral(1), + field("@timestamp"))))))); + } + + @Test + public void testTimechartWithPerHourFunction() { + assertEqual( + "source=t | timechart per_hour(a)", + eval( + new Timechart(relation("t"), alias("per_hour(a)", aggregate("sum", field("a")))) + .span(span(field("@timestamp"), intLiteral(1), SpanUnit.of("m"))) + .limit(10) + .useOther(true), + let( + field("per_hour(a)"), + function( + "/", + function("*", field("per_hour(a)"), doubleLiteral(3600000.0)), + function( + "timestampdiff", + stringLiteral("MILLISECOND"), + field("@timestamp"), + function( + "timestampadd", + stringLiteral("MINUTE"), + intLiteral(1), + field("@timestamp"))))))); + } + + @Test + public void testTimechartWithPerDayFunction() { + assertEqual( + "source=t | timechart per_day(a)", + eval( + new Timechart(relation("t"), alias("per_day(a)", aggregate("sum", field("a")))) + .span(span(field("@timestamp"), intLiteral(1), SpanUnit.of("m"))) + .limit(10) + .useOther(true), + let( + field("per_day(a)"), + function( + "/", + function("*", field("per_day(a)"), doubleLiteral(8.64E7)), + function( + "timestampdiff", + stringLiteral("MILLISECOND"), + field("@timestamp"), + function( + "timestampadd", + stringLiteral("MINUTE"), + intLiteral(1), + field("@timestamp"))))))); + } + + @Test + public void testStatsWithPerSecondThrowsException() { + assertEquals( + "per_second function can only be used within timechart command", + assertThrows(SyntaxCheckException.class, () -> plan("source=t | stats per_second(a)")) + .getMessage()); + assertEquals( + "per_minute function can only be used within timechart command", + assertThrows(SyntaxCheckException.class, () -> plan("source=t | stats per_minute(a)")) + .getMessage()); + assertEquals( + "per_hour function can only be used within timechart command", + assertThrows(SyntaxCheckException.class, () -> plan("source=t | stats per_hour(a)")) + .getMessage()); + assertEquals( + "per_day function can only be used within timechart command", + assertThrows(SyntaxCheckException.class, () -> plan("source=t | stats per_day(a)")) + .getMessage()); + } + protected void assertEqual(String query, Node expectedPlan) { Node actualPlan = plan(query); assertEquals(expectedPlan, actualPlan); @@ -1106,4 +1372,120 @@ public void testRexSedModeWithOffsetFieldThrowsException() { // Test that SED mode and offset_field cannot be used together (align with Splunk behavior) plan("source=test | rex field=email mode=sed offset_field=matchpos \"s/@.*/@company.com/\""); } + + // Multisearch tests + + @Test + public void testBasicMultisearchParsing() { + // Test basic multisearch parsing + plan("| multisearch [ search source=test1 ] [ search source=test2 ]"); + } + + @Test + public void testMultisearchWithStreamingCommands() { + // Test multisearch with streaming commands + plan( + "| multisearch [ search source=test1 | where age > 30 | fields name, age ] " + + "[ search source=test2 | eval category=\"young\" | rename id as user_id ]"); + } + + @Test + public void testMultisearchWithStatsCommand() { + // Test multisearch with stats command - now allowed + plan( + "| multisearch [ search source=test1 | stats count() by gender ] " + + "[ search source=test2 | fields name, age ]"); + } + + @Test + public void testMultisearchWithSortCommand() { + // Test multisearch with sort command - now allowed + plan( + "| multisearch [ search source=test1 | sort age ] " + + "[ search source=test2 | fields name, age ]"); + } + + @Test + public void testMultisearchWithBinCommand() { + // Test multisearch with bin command - now allowed + plan( + "| multisearch [ search source=test1 | bin age span=10 ] " + + "[ search source=test2 | fields name, age ]"); + } + + @Test + public void testMultisearchWithTimechartCommand() { + // Test multisearch with timechart command - now allowed + plan( + "| multisearch [ search source=test1 | timechart count() by age ] " + + "[ search source=test2 | fields name, age ]"); + } + + @Test + public void testMultisearchWithRareCommand() { + // Test multisearch with rare command - now allowed + plan( + "| multisearch [ search source=test1 | rare gender ] " + + "[ search source=test2 | fields name, age ]"); + } + + @Test + public void testMultisearchWithDedupeCommand() { + // Test multisearch with dedup command - now allowed + plan( + "| multisearch [ search source=test1 | dedup name ] " + + "[ search source=test2 | fields name, age ]"); + } + + @Test + public void testMultisearchWithJoinCommand() { + // Test multisearch with join command - now allowed + plan( + "| multisearch [ search source=test1 | join left=l right=r where l.id = r.id" + + " test2 ] [ search source=test3 | fields name, age ]"); + } + + @Test + public void testMultisearchWithComplexPipeline() { + // Test multisearch with complex pipeline (previously called streaming) + plan( + "| multisearch [ search source=test1 | where age > 30 | eval category=\"adult\"" + + " | fields name, age, category | rename age as years_old | head 100 ] [ search" + + " source=test2 | where status=\"active\" | expand tags | flatten nested_data |" + + " fillnull with \"unknown\" | reverse ]"); + } + + @Test + public void testMultisearchMixedCommands() { + // Test multisearch with mix of commands - now all allowed + plan( + "| multisearch [ search source=test1 | where age > 30 | stats count() ] " + + "[ search source=test2 | where status=\"active\" | sort name ]"); + } + + @Test + public void testMultisearchSingleSubsearchThrowsException() { + // Test multisearch with only one subsearch - should throw descriptive runtime exception + SyntaxCheckException exception = + assertThrows( + SyntaxCheckException.class, + () -> plan("| multisearch [ search source=test1 | fields name, age ]")); + + // Now we should get our descriptive runtime validation error message + assertEquals( + "Multisearch command requires at least two subsearches. Provided: 1", + exception.getMessage()); + } + + @Test + public void testReplaceCommand() { + // Test basic single pattern replacement + plan("source=t | replace 'old' WITH 'new' IN field"); + } + + @Test + public void testReplaceCommandWithMultiplePairs() { + // Test multiple pattern/replacement pairs + plan("source=t | replace 'a' WITH 'A', 'b' WITH 'B' IN field"); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java index e57b3fab4ea..6b0e0a081f8 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -11,6 +11,7 @@ import static org.opensearch.sql.ast.dsl.AstDSL.agg; import static org.opensearch.sql.ast.dsl.AstDSL.aggregate; import static org.opensearch.sql.ast.dsl.AstDSL.alias; +import static org.opensearch.sql.ast.dsl.AstDSL.allFields; import static org.opensearch.sql.ast.dsl.AstDSL.and; import static org.opensearch.sql.ast.dsl.AstDSL.argument; import static org.opensearch.sql.ast.dsl.AstDSL.booleanLiteral; @@ -43,6 +44,7 @@ import static org.opensearch.sql.ast.dsl.AstDSL.relation; import static org.opensearch.sql.ast.dsl.AstDSL.search; import static org.opensearch.sql.ast.dsl.AstDSL.sort; +import static org.opensearch.sql.ast.dsl.AstDSL.span; import static org.opensearch.sql.ast.dsl.AstDSL.stringLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.unresolvedArg; import static org.opensearch.sql.ast.dsl.AstDSL.when; @@ -59,6 +61,9 @@ import org.opensearch.sql.ast.expression.AllFields; import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.RelevanceFieldList; +import org.opensearch.sql.ast.expression.SpanUnit; +import org.opensearch.sql.ast.tree.Timechart; +import org.opensearch.sql.calcite.plan.OpenSearchConstants; import org.opensearch.sql.common.antlr.SyntaxCheckException; public class AstExpressionBuilderTest extends AstBuilderTest { @@ -1384,4 +1389,220 @@ public void testTimeModifierEarliestWithStringValue() { "source=t earliest='2025-12-10 14:00:00'", search(relation("t"), "@timestamp:>=2025\\-12\\-10T14\\:00\\:00Z")); } + + @Test + public void testTimechartSpanParameter() { + assertEqual( + "source=t | timechart span=30m count()", + Timechart.builder() + .child(relation("t")) + .binExpression( + span( + field(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP), + intLiteral(30), + SpanUnit.m)) + .aggregateFunction(aggregate("count", allFields())) + .limit(10) + .useOther(true) + .build()); + } + + @Test + public void testTimechartLimitParameter() { + assertEqual( + "source=t | timechart limit=100 count()", + Timechart.builder() + .child(relation("t")) + .binExpression( + span( + field(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP), intLiteral(1), SpanUnit.m)) + .aggregateFunction(aggregate("count", allFields())) + .limit(100) + .useOther(true) + .build()); + } + + @Test + public void testTimechartNegativeLimitParameter() { + assertThrows( + IllegalArgumentException.class, + () -> assertEqual("source=t | timechart limit=-1 count()", (Node) null)); + } + + @Test + public void testTimechartUseOtherWithBooleanLiteral() { + assertEqual( + "source=t | timechart useother=true count()", + Timechart.builder() + .child(relation("t")) + .binExpression( + span( + field(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP), intLiteral(1), SpanUnit.m)) + .aggregateFunction(aggregate("count", allFields())) + .limit(10) + .useOther(true) + .build()); + + assertEqual( + "source=t | timechart useother=false count()", + Timechart.builder() + .child(relation("t")) + .binExpression( + span( + field(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP), intLiteral(1), SpanUnit.m)) + .aggregateFunction(aggregate("count", allFields())) + .limit(10) + .useOther(false) + .build()); + } + + @Test + public void testTimechartUseOtherWithIdentifier() { + assertEqual( + "source=t | timechart useother=t count()", + Timechart.builder() + .child(relation("t")) + .binExpression( + span( + field(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP), intLiteral(1), SpanUnit.m)) + .aggregateFunction(aggregate("count", allFields())) + .limit(10) + .useOther(true) + .build()); + + assertEqual( + "source=t | timechart useother=f count()", + Timechart.builder() + .child(relation("t")) + .binExpression( + span( + field(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP), intLiteral(1), SpanUnit.m)) + .aggregateFunction(aggregate("count", allFields())) + .limit(10) + .useOther(false) + .build()); + + assertEqual( + "source=t | timechart useother=TRUE count()", + Timechart.builder() + .child(relation("t")) + .binExpression( + span( + field(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP), intLiteral(1), SpanUnit.m)) + .aggregateFunction(aggregate("count", allFields())) + .limit(10) + .useOther(true) + .build()); + + assertEqual( + "source=t | timechart useother=FALSE count()", + Timechart.builder() + .child(relation("t")) + .binExpression( + span( + field(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP), intLiteral(1), SpanUnit.m)) + .aggregateFunction(aggregate("count", allFields())) + .limit(10) + .useOther(false) + .build()); + } + + @Test + public void testTimechartInvalidUseOtherValue() { + assertThrows( + IllegalArgumentException.class, + () -> assertEqual("source=t | timechart useother=invalid count()", (Node) null)); + } + + @Test + public void testTimechartInvalidParameter() { + assertThrows( + SyntaxCheckException.class, + () -> assertEqual("source=t | timechart invalidparam=value count()", (Node) null)); + } + + @Test + public void testVisitSpanClause() { + // Test span clause with explicit field + assertEqual( + "source=t | stats count() by span(timestamp, 1h)", + agg( + relation("t"), + exprList(alias("count()", aggregate("count", AllFields.of()))), + emptyList(), + emptyList(), + alias("span(timestamp,1h)", span(field("timestamp"), intLiteral(1), SpanUnit.H)), + defaultStatsArgs())); + + // Test span clause with different time unit + assertEqual( + "source=t | stats count() by span(timestamp, 5d)", + agg( + relation("t"), + exprList(alias("count()", aggregate("count", AllFields.of()))), + emptyList(), + emptyList(), + alias("span(timestamp,5d)", span(field("timestamp"), intLiteral(5), SpanUnit.D)), + defaultStatsArgs())); + + // Test span clause with implicit @timestamp field + assertEqual( + "source=t | stats count() by span(1m)", + agg( + relation("t"), + exprList(alias("count()", aggregate("count", AllFields.of()))), + emptyList(), + emptyList(), + alias( + "span(1m)", + span( + field(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP), + intLiteral(1), + SpanUnit.m)), + defaultStatsArgs())); + } + + @Test + public void testVisitSpanLiteral() { + // Test span literal with integer value and hour unit + assertEqual( + "source=t | timechart span=1h count()", + Timechart.builder() + .child(relation("t")) + .binExpression( + span( + field(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP), intLiteral(1), SpanUnit.H)) + .aggregateFunction(aggregate("count", allFields())) + .limit(10) + .useOther(true) + .build()); + + // Test span literal with decimal value and minute unit + assertEqual( + "source=t | timechart span=2m count()", + Timechart.builder() + .child(relation("t")) + .binExpression( + span( + field(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP), intLiteral(2), SpanUnit.m)) + .aggregateFunction(aggregate("count", allFields())) + .limit(10) + .useOther(true) + .build()); + + // Test span literal without unit (should use NONE unit) + assertEqual( + "source=t | timechart span=10 count()", + Timechart.builder() + .child(relation("t")) + .binExpression( + span( + field(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP), + intLiteral(10), + SpanUnit.NONE)) + .aggregateFunction(aggregate("count", allFields())) + .limit(10) + .useOther(true) + .build()); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 07bc711056a..48f6c45b4c6 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -173,6 +173,34 @@ public void testEventstatsCommandWithSpanFunction() { anonymize("source=t | eventstats count(a) by span(b, 1d), c")); } + @Test + public void testStreamstatsCommandWithByClause() { + assertEquals( + "source=table | streamstats count(identifier) by identifier", + anonymize("source=t | streamstats count(a) by b")); + } + + @Test + public void testStreamstatsCommandWithWindowAndCurrent() { + assertEquals( + "source=table | streamstats max(identifier)", + anonymize("source=t | streamstats current=false window=2 max(a)")); + } + + @Test + public void testStreamstatsCommandWithNestedFunctions() { + assertEquals( + "source=table | streamstats sum(+(identifier,identifier))", + anonymize("source=t | streamstats sum(a+b)")); + } + + @Test + public void testStreamstatsCommandWithSpanFunction() { + assertEquals( + "source=table | streamstats count(identifier) by span(identifier, *** d),identifier", + anonymize("source=t | streamstats count(a) by span(b, 1d), c")); + } + @Test public void testBinCommandBasic() { assertEquals("source=table | bin identifier span=***", anonymize("source=t | bin f span=10")); @@ -281,6 +309,18 @@ public void testFillNullWithoutFields() { assertEquals("source=table | fillnull with ***", anonymize("source=t | fillnull with 0")); } + @Test + public void testFillNullValueSyntaxWithFields() { + assertEquals( + "source=table | fillnull value=*** identifier identifier", + anonymize("source=t | fillnull value=0 f1 f2")); + } + + @Test + public void testFillNullValueSyntaxAllFields() { + assertEquals("source=table | fillnull value=***", anonymize("source=t | fillnull value=0")); + } + @Test public void testRareCommandWithGroupBy() { when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(false); @@ -299,7 +339,8 @@ public void testTopCommandWithNAndGroupBy() { public void testRareCommandWithGroupByWithCalcite() { when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(true); assertEquals( - "source=table | rare 10 countield='count' showcount=true identifier by identifier", + "source=table | rare 10 countield='count' showcount=true usenull=true identifier by" + + " identifier", anonymize("source=t | rare a by b")); } @@ -307,7 +348,8 @@ public void testRareCommandWithGroupByWithCalcite() { public void testTopCommandWithNAndGroupByWithCalcite() { when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(true); assertEquals( - "source=table | top 1 countield='count' showcount=true identifier by identifier", + "source=table | top 1 countield='count' showcount=true usenull=true identifier by" + + " identifier", anonymize("source=t | top 1 a by b")); } @@ -605,6 +647,49 @@ public void testGrok() { anonymize("source=t | grok email '.+@%{HOSTNAME:host}' | fields email, host")); } + @Test + public void testReplaceCommandSingleField() { + assertEquals( + "source=table | replace *** WITH *** IN Field(field=fieldname, fieldArgs=[])", + anonymize("source=EMP | replace \"value\" WITH \"newvalue\" IN fieldname")); + } + + @Test + public void testReplaceCommandMultipleFields() { + assertEquals( + "source=table | replace *** WITH *** IN Field(field=fieldname1, fieldArgs=[])," + + " Field(field=fieldname2, fieldArgs=[])", + anonymize("source=EMP | replace \"value\" WITH \"newvalue\" IN fieldname1, fieldname2")); + } + + @Test(expected = Exception.class) + public void testReplaceCommandWithoutInShouldFail() { + anonymize("source=EMP | replace \"value\" WITH \"newvalue\""); + } + + @Test + public void testReplaceCommandSpecialCharactersInFields() { + assertEquals( + "source=table | replace *** WITH *** IN Field(field=user.name, fieldArgs=[])," + + " Field(field=user.email, fieldArgs=[])", + anonymize("source=EMP | replace \"value\" WITH \"newvalue\" IN user.name, user.email")); + } + + @Test + public void testReplaceCommandWithWildcards() { + assertEquals( + "source=table | replace *** WITH *** IN Field(field=fieldname, fieldArgs=[])", + anonymize("source=EMP | replace \"CLERK*\" WITH \"EMPLOYEE*\" IN fieldname")); + } + + @Test + public void testReplaceCommandWithMultipleWildcards() { + assertEquals( + "source=table | replace *** WITH *** IN Field(field=fieldname1, fieldArgs=[])," + + " Field(field=fieldname2, fieldArgs=[])", + anonymize("source=EMP | replace \"*TEST*\" WITH \"*NEW*\" IN fieldname1, fieldname2")); + } + @Test public void testPatterns() { when(settings.getSettingValue(Key.PATTERN_METHOD)).thenReturn("SIMPLE_PATTERN"); @@ -676,6 +761,13 @@ public void testMvjoin() { anonymize("source=t | eval result=mvjoin(array('a', 'b', 'c'), ',') | fields result")); } + @Test + public void testMvappend() { + assertEquals( + "source=table | eval identifier=mvappend(identifier,***,***) | fields + identifier", + anonymize("source=t | eval result=mvappend(a, 'b', 'c') | fields result")); + } + @Test public void testRexWithOffsetField() { when(settings.getSettingValue(Key.PPL_REX_MAX_MATCH_LIMIT)).thenReturn(10); @@ -686,6 +778,30 @@ public void testRexWithOffsetField() { anonymize("source=t | rex field=message \"(?[a-z]+)\" offset_field=pos")); } + @Test + public void testMultisearch() { + assertEquals( + "| multisearch [search source=table | where identifier < ***] [search" + + " source=table | where identifier >= ***]", + anonymize( + "| multisearch [search source=accounts | where age < 30] [search" + + " source=accounts | where age >= 30]")); + + assertEquals( + "| multisearch [search source=table | where identifier > ***] [search" + + " source=table | where identifier = ***]", + anonymize( + "| multisearch [search source=accounts | where balance > 20000]" + + " [search source=accounts | where state = 'CA']")); + + assertEquals( + "| multisearch [search source=table | fields + identifier,identifier] [search" + + " source=table | where identifier = ***]", + anonymize( + "| multisearch [search source=accounts | fields firstname, lastname]" + + " [search source=accounts | where age = 25]")); + } + private String anonymize(String query) { AstBuilder astBuilder = new AstBuilder(query, settings); return anonymize(astBuilder.visit(parser.parse(query))); @@ -712,4 +828,13 @@ public void testSearchWithAbsoluteTimeRange() { "source=table (@timestamp:*** AND (@timestamp:***", anonymize("search source=t earliest='2012-12-10 15:00:00' latest=now")); } + + @Test + public void testSpath() { + assertEquals( + "source=table | spath input=identifier output=identifier path=identifier | fields +" + + " identifier,identifier", + anonymize( + "search source=t | spath input=json_attr output=out path=foo.bar | fields id, out")); + } } diff --git a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/Format.java b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/Format.java index d5537ff5556..14204554ff3 100644 --- a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/Format.java +++ b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/Format.java @@ -22,7 +22,9 @@ public enum Format { SIMPLE("simple"), STANDARD("standard"), EXTENDED("extended"), - COST("cost"); + COST("cost"), + /** Returns explain output in yaml format */ + YAML("yaml"); @Getter private final String formatName; @@ -44,6 +46,7 @@ public enum Format { builder.put(STANDARD.formatName, STANDARD); builder.put(EXTENDED.formatName, EXTENDED); builder.put(COST.formatName, COST); + builder.put(YAML.formatName, YAML); EXPLAIN_FORMATS = builder.build(); } diff --git a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/YamlResponseFormatter.java b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/YamlResponseFormatter.java new file mode 100644 index 00000000000..da3022fbc82 --- /dev/null +++ b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/YamlResponseFormatter.java @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.protocol.response.format; + +import java.security.AccessController; +import java.security.PrivilegedAction; +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.utils.YamlFormatter; + +/** + * Abstract class for all YAML formatter. + * + * @param response generic type which could be DQL or DML response + */ +@RequiredArgsConstructor +public abstract class YamlResponseFormatter implements ResponseFormatter { + + public static final String CONTENT_TYPE = "application/yaml; charset=UTF-8"; + + @Override + public String format(R response) { + return yamlify(buildYamlObject(response)); + } + + @Override + public String format(Throwable t) { + return yamlify(t); + } + + public String contentType() { + return CONTENT_TYPE; + } + + /** + * Build YAML object to generate response yaml string. + * + * @param response response + * @return yaml object for response + */ + protected abstract Object buildYamlObject(R response); + + protected String yamlify(Object yamlObject) { + return AccessController.doPrivileged( + (PrivilegedAction) () -> YamlFormatter.formatToYaml(yamlObject)); + } +} diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/FormatTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/FormatTest.java index e37823ce829..fbe12ef44b3 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/FormatTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/FormatTest.java @@ -50,6 +50,13 @@ void extended() { assertEquals(Format.EXTENDED, format.get()); } + @Test + void yaml() { + Optional format = Format.ofExplain("yaml"); + assertTrue(format.isPresent()); + assertEquals(Format.YAML, format.get()); + } + @Test void defaultExplainFormat() { Optional format = Format.ofExplain(""); diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/YamlResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/YamlResponseFormatterTest.java new file mode 100644 index 00000000000..9710dde0a88 --- /dev/null +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/YamlResponseFormatterTest.java @@ -0,0 +1,55 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.protocol.response.format; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.LinkedHashMap; +import java.util.Map; +import org.junit.jupiter.api.DisplayNameGeneration; +import org.junit.jupiter.api.DisplayNameGenerator; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.utils.YamlFormatter; + +@DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) +class YamlResponseFormatterTest { + + private final YamlResponseFormatter formatter = + new YamlResponseFormatter<>() { + @Override + protected Object buildYamlObject(Object response) { + // Pass-through for testing: return the response directly + return response; + } + }; + + @Test + void content_type_matches_yaml() { + assertEquals(YamlResponseFormatter.CONTENT_TYPE, formatter.contentType()); + } + + @Test + void formats_response_via_yaml_formatter() { + Map payload = new LinkedHashMap<>(); + payload.put("b", 2); + payload.put("a", "1"); + + String expected = YamlFormatter.formatToYaml(payload); + String actual = formatter.format(payload); + + assertEquals(expected, actual); + } + + @Test + void formats_throwable_via_yaml_formatter() { + Exception e = new Exception("boom", new RuntimeException("root-cause")); + + String expected = YamlFormatter.formatToYaml(e); + String actual = formatter.format(e); + + assertEquals(expected, actual); + } +} diff --git a/scripts/bwctest.sh b/scripts/bwctest-full-restart.sh similarity index 96% rename from scripts/bwctest.sh rename to scripts/bwctest-full-restart.sh index 4f017ae5e89..5709a549b92 100755 --- a/scripts/bwctest.sh +++ b/scripts/bwctest-full-restart.sh @@ -54,5 +54,5 @@ function setup_bwc_artifact() { } setup_bwc_artifact -./gradlew bwcTestSuite -Dtests.security.manager=false +./gradlew bwcTestFullRestartSuite -Dtests.security.manager=false diff --git a/scripts/bwctest-rolling-upgrade.sh b/scripts/bwctest-rolling-upgrade.sh new file mode 100755 index 00000000000..a45f5dd2bbd --- /dev/null +++ b/scripts/bwctest-rolling-upgrade.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +set -e + +function usage() { + echo "" + echo "This script is used to run Backwards Compatibility tests" + echo "--------------------------------------------------------------------------" + echo "Usage: $0 [args]" + echo "" + echo "Required arguments:" + echo "None" + echo "" + echo -e "-h\tPrint this message." + echo "--------------------------------------------------------------------------" +} + +while getopts ":h" arg; do + case $arg in + h) + usage + exit 1 + ;; + ?) + echo "Invalid option: -${OPTARG}" + exit 1 + ;; + esac +done + +# Place SQL artifact for the current version for bwc +function setup_bwc_artifact() { + # This gets opensearch version from build.gradle (e.g. 1.2.0-SNAPSHOT), + # then converts to plugin version by appending ".0" (e.g. 1.2.0.0-SNAPSHOT), + # assuming one line in build.gradle is 'opensearch_version = System.getProperty("opensearch.version", "")'. + plugin_version=$(grep 'opensearch_version = System.getProperty' build.gradle | \ + grep -o '[0-9]\+\.[0-9]\+\.[0-9]\+[^"]*' | sed -e 's/\(.*\)\(\.[0-9]\)/\1\2.0/') + plugin_artifact="./plugin/build/distributions/opensearch-sql-$plugin_version.zip" + bwc_artifact_dir="./integ-test/src/test/resources/bwc/$plugin_version" + + if [ -z "${plugin_version// }" ]; then + echo "Error: failed to retrieve plugin version from build.gradle." >&2 + exit 1 + fi + + # copy current artifact to bwc artifact directory if it's not there + if [ ! -f "$bwc_artifact_dir/opensearch-sql-$plugin_version.zip" ]; then + if [ ! -f "$plugin_artifact" ]; then + ./gradlew assemble + fi + mkdir -p "$bwc_artifact_dir" + cp "$plugin_artifact" "$bwc_artifact_dir" + fi +} + +setup_bwc_artifact +./gradlew bwcTestRollingUpgradeSuite -Dtests.security.manager=false + diff --git a/settings.gradle b/settings.gradle index 0e88df0b0df..4b0fa1b44f8 100644 --- a/settings.gradle +++ b/settings.gradle @@ -8,6 +8,7 @@ rootProject.name = 'opensearch-sql' include 'opensearch-sql-plugin' project(':opensearch-sql-plugin').projectDir = file('plugin') +include 'api' include 'ppl' include 'common' include 'opensearch'