diff --git a/opensearch/README.md b/opensearch/README.md new file mode 100644 index 000000000..a1c10be30 --- /dev/null +++ b/opensearch/README.md @@ -0,0 +1,47 @@ +# Plugin Module: opensearch + +Datasource plugin for [OpenSearch](https://opensearch.org/) in Perses. Supports log queries using +[PPL (Piped Processing Language)](https://opensearch.org/docs/latest/search-plugins/sql/ppl/index/). + +This plugin is intended to display logs alongside the traces surfaced by the Tempo plugin, so a user can +pivot from a trace to the related logs in OpenSearch. + +### How to install + +This plugin requires react and react-dom 18. + +```bash +npm install react@18 react-dom@18 +npm install @perses-dev/opensearch-plugin +``` + +## Development + +```bash +npm install +npm run dev # start the dev server +npm run build # build the plugin +npm test # run unit tests +``` + +## Trace to logs + +This plugin is intended to display logs alongside traces produced by the Tempo or +Jaeger plugins. Use a dashboard variable (e.g. `$traceId`) to share the selected +trace between the trace panel and the OpenSearch logs panel. + +PPL example: + +``` +source=otel-logs-* | where traceId='$traceId' +``` + +A complete example dashboard is in `docs/examples/trace-to-logs.json`. See +`docs/examples/README.md` for field-name conventions and how to swap Tempo for Jaeger. + +## Field overrides + +OpenSearch is schema-flexible; field names vary by exporter. The plugin tries +common names by default (`@timestamp`/`timestamp`/`time` for the timestamp; +`message`/`log`/`body` for the message). If your index uses different names, +set `timestampField` and `messageField` on the query spec. diff --git a/opensearch/cue.mod/module.cue b/opensearch/cue.mod/module.cue new file mode 100644 index 000000000..c8dddba8b --- /dev/null +++ b/opensearch/cue.mod/module.cue @@ -0,0 +1,13 @@ +module: "github.com/perses/plugins/opensearch@v0" +language: { + version: "v0.15.1" +} +source: { + kind: "git" +} +deps: { + "github.com/perses/shared/cue@v0": { + v: "v0.53.1" + default: true + } +} diff --git a/opensearch/docs/examples/README.md b/opensearch/docs/examples/README.md new file mode 100644 index 000000000..90c323f81 --- /dev/null +++ b/opensearch/docs/examples/README.md @@ -0,0 +1,30 @@ +# OpenSearch examples + +## trace-to-logs.json + +Demonstrates the trace→logs pivot: + +1. The `traceId` dashboard variable holds the currently selected trace. +2. The Tempo panel renders the trace via `TempoTraceQuery` and `TracingGanttChart`. +3. The OpenSearch logs panel runs a PPL query filtered by `traceId='$traceId'`. + +When the user picks a trace in the gantt chart (or sets the variable elsewhere), +the logs panel re-runs and shows logs for that trace. + +### Field-name conventions + +OpenSearch indexes vary in how they name fields. Common defaults: + +| Source | timestamp | message | trace id | +| ---------------------------- | ------------ | --------- | ---------- | +| OpenTelemetry / Data Prepper | `@timestamp` | `body` | `traceId` | +| AWS / X-Ray exporter | `@timestamp` | `message` | `traceId` | +| Legacy / custom | `time` | `message` | `trace_id` | + +If your index uses different names, set `timestampField` and/or `messageField` on +the `OpenSearchLogQuery` spec, and adjust the `where` clause to your trace_id field. + +### Swap Tempo for Jaeger + +Replace the Tempo panel's `plugin.kind` and `datasource.kind` with `JaegerTraceQuery` +and `JaegerDatasource`. The OpenSearch panel does not change. diff --git a/opensearch/docs/examples/docker-compose.yml b/opensearch/docs/examples/docker-compose.yml new file mode 100644 index 000000000..7d1a9cbc5 --- /dev/null +++ b/opensearch/docs/examples/docker-compose.yml @@ -0,0 +1,42 @@ +services: + opensearch: + image: opensearchproject/opensearch:2 + container_name: opensearch + environment: + - discovery.type=single-node + - DISABLE_SECURITY_PLUGIN=true + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=Perses-Test-1! + - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m + - bootstrap.memory_lock=true + - http.cors.enabled=true + - http.cors.allow-origin=* + - http.cors.allow-methods=GET,POST,PUT,DELETE,OPTIONS + - http.cors.allow-headers=X-Requested-With,Content-Type,Content-Length,Authorization + - http.cors.allow-credentials=true + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + ports: + - "9200:9200" + - "9600:9600" + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:9200/_cluster/health || exit 1"] + interval: 5s + timeout: 5s + retries: 20 + + dashboards: + image: opensearchproject/opensearch-dashboards:2 + container_name: opensearch-dashboards + environment: + - OPENSEARCH_HOSTS=["http://opensearch:9200"] + - DISABLE_SECURITY_DASHBOARDS_PLUGIN=true + ports: + - "5601:5601" + depends_on: + opensearch: + condition: service_healthy diff --git a/opensearch/docs/examples/trace-to-logs.json b/opensearch/docs/examples/trace-to-logs.json new file mode 100644 index 000000000..fdfd26792 --- /dev/null +++ b/opensearch/docs/examples/trace-to-logs.json @@ -0,0 +1,84 @@ +{ + "kind": "Dashboard", + "metadata": { + "name": "trace-to-logs-example", + "project": "examples" + }, + "spec": { + "display": { + "name": "Trace to Logs (Tempo + OpenSearch)" + }, + "duration": "1h", + "variables": [ + { + "kind": "TextVariable", + "spec": { + "name": "traceId", + "value": "" + } + } + ], + "panels": { + "trace": { + "kind": "Panel", + "spec": { + "display": { "name": "Trace" }, + "plugin": { + "kind": "TracingGanttChart", + "spec": {} + }, + "queries": [ + { + "kind": "TraceQuery", + "spec": { + "plugin": { + "kind": "TempoTraceQuery", + "spec": { + "query": "$traceId", + "datasource": { "kind": "TempoDatasource", "name": "tempo" } + } + } + } + } + ] + } + }, + "logs": { + "kind": "Panel", + "spec": { + "display": { "name": "Logs for current trace" }, + "plugin": { + "kind": "LogsTable", + "spec": {} + }, + "queries": [ + { + "kind": "LogQuery", + "spec": { + "plugin": { + "kind": "OpenSearchLogQuery", + "spec": { + "datasource": { "kind": "OpenSearchDatasource", "name": "opensearch" }, + "index": "otel-logs-*", + "query": "source=otel-logs-* | where traceId='$traceId'" + } + } + } + } + ] + } + } + }, + "layouts": [ + { + "kind": "Grid", + "spec": { + "items": [ + { "x": 0, "y": 0, "width": 24, "height": 12, "content": { "$ref": "#/spec/panels/trace" } }, + { "x": 0, "y": 12, "width": 24, "height": 12, "content": { "$ref": "#/spec/panels/logs" } } + ] + } + } + ] + } +} diff --git a/opensearch/go.mod b/opensearch/go.mod new file mode 100644 index 000000000..a3d976f2c --- /dev/null +++ b/opensearch/go.mod @@ -0,0 +1,32 @@ +module github.com/perses/plugins/opensearch + +go 1.25.7 + +require github.com/perses/perses v0.53.1 + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/go-jose/go-jose/v4 v4.1.3 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/jpillora/backoff v1.0.0 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/muhlemmer/gu v0.3.1 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect + github.com/perses/common v0.30.2 // indirect + github.com/prometheus/client_golang v1.23.2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.67.5 // indirect + github.com/prometheus/procfs v0.17.0 // indirect + github.com/zitadel/oidc/v3 v3.45.4 // indirect + github.com/zitadel/schema v1.3.2 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + golang.org/x/net v0.49.0 // indirect + golang.org/x/oauth2 v0.35.0 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/text v0.34.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/opensearch/go.sum b/opensearch/go.sum new file mode 100644 index 000000000..1d709f9ee --- /dev/null +++ b/opensearch/go.sum @@ -0,0 +1,70 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= +github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/muhlemmer/gu v0.3.1 h1:7EAqmFrW7n3hETvuAdmFmn4hS8W+z3LgKtrnow+YzNM= +github.com/muhlemmer/gu v0.3.1/go.mod h1:YHtHR+gxM+bKEIIs7Hmi9sPT3ZDUvTN/i88wQpZkrdM= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nexucis/lamenv v0.5.2 h1:tK/u3XGhCq9qIoVNcXsK9LZb8fKopm0A5weqSRvHd7M= +github.com/nexucis/lamenv v0.5.2/go.mod h1:HusJm6ltmmT7FMG8A750mOLuME6SHCsr2iFYxp5fFi0= +github.com/perses/common v0.30.2 h1:RAiVxUpX76lTCb4X7pfcXSvYdXQmZwKi4oDKAEO//u0= +github.com/perses/common v0.30.2/go.mod h1:DFtur1QPah2/ChXbKKhw7djYdwNgz27s5fPKpiK0Xao= +github.com/perses/perses v0.53.1 h1:9VY/6p9QWrZwPSV7qiwTMSOsgcB37Lb1AXKT0ORXc6I= +github.com/perses/perses v0.53.1/go.mod h1:ro8fsgBkHYOdrL/MV+fdP9mflKzYCy/+gcbxiaReI/A= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= +github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= +github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= +github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/zitadel/oidc/v3 v3.45.4 h1:GKyWaPRVQ8sCu9XgJ3NgNGtG52FzwVJpzXjIUG2+YrI= +github.com/zitadel/oidc/v3 v3.45.4/go.mod h1:XALmFXS9/kSom9B6uWin1yJ2WTI/E4Ti5aXJdewAVEs= +github.com/zitadel/schema v1.3.2 h1:gfJvt7dOMfTmxzhscZ9KkapKo3Nei3B6cAxjav+lyjI= +github.com/zitadel/schema v1.3.2/go.mod h1:IZmdfF9Wu62Zu6tJJTH3UsArevs3Y4smfJIj3L8fzxw= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= +golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/opensearch/jest.config.ts b/opensearch/jest.config.ts new file mode 100644 index 000000000..bc094df8a --- /dev/null +++ b/opensearch/jest.config.ts @@ -0,0 +1,31 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import type { Config } from '@jest/types'; +import shared from '../jest.shared'; + +const jestConfig: Config.InitialOptions = { + ...shared, + + moduleNameMapper: { + ...(shared.moduleNameMapper ?? {}), + '^react$': '/../node_modules/react', + '^react-dom$': '/../node_modules/react-dom', + '^react/jsx-runtime$': '/../node_modules/react/jsx-runtime', + '^react-dom/(.*)$': '/../node_modules/react-dom/$1', + }, + + setupFilesAfterEnv: [...(shared.setupFilesAfterEnv ?? []), '/src/setup-tests.ts'], +}; + +export default jestConfig; diff --git a/opensearch/package.json b/opensearch/package.json new file mode 100644 index 000000000..a6696ff58 --- /dev/null +++ b/opensearch/package.json @@ -0,0 +1,68 @@ +{ + "name": "@perses-dev/opensearch-plugin", + "version": "0.1.0", + "scripts": { + "dev": "rsbuild dev", + "build": "npm run build-mf && concurrently \"npm:build:*\"", + "build-mf": "rsbuild build", + "build:cjs": "swc ./src -d dist/lib/cjs --strip-leading-paths --config-file ../.cjs.swcrc", + "build:esm": "swc ./src -d dist/lib --strip-leading-paths --config-file ../.swcrc", + "build:types": "tsc --project tsconfig.build.json", + "lint": "eslint src --ext .ts,.tsx", + "test": "cross-env LC_ALL=C TZ=UTC jest", + "type-check": "tsc --noEmit" + }, + "main": "lib/cjs/index.js", + "module": "lib/index.js", + "types": "lib/index.d.ts", + "peerDependencies": { + "@emotion/react": "^11.7.1", + "@emotion/styled": "^11.6.0", + "@hookform/resolvers": "^3.2.0", + "@perses-dev/components": "^0.54.0-beta.3", + "@perses-dev/dashboards": "^0.54.0-beta.3", + "@perses-dev/explore": "^0.54.0-beta.3", + "@perses-dev/plugin-system": "^0.54.0-beta.3", + "@perses-dev/spec": "^0.2.0-beta.2", + "@tanstack/react-query": "^4.39.1", + "date-fns": "^4.1.0", + "date-fns-tz": "^3.2.0", + "echarts": "5.5.0", + "immer": "^10.1.1", + "lodash": "^4.17.21", + "react": "^17.0.2 || ^18.0.0", + "react-dom": "^17.0.2 || ^18.0.0", + "react-hook-form": "^7.52.2", + "use-resize-observer": "^9.0.0" + }, + "files": [ + "lib/**/*", + "__mf/**/*", + "mf-manifest.json", + "mf-stats.json" + ], + "perses": { + "moduleName": "OpenSearch", + "schemasPath": "schemas", + "plugins": [ + { + "kind": "Datasource", + "spec": { + "display": { + "name": "OpenSearch Datasource" + }, + "name": "OpenSearchDatasource" + } + }, + { + "kind": "LogQuery", + "spec": { + "display": { + "name": "OpenSearch Log Query" + }, + "name": "OpenSearchLogQuery" + } + } + ] + } +} diff --git a/opensearch/rsbuild.config.ts b/opensearch/rsbuild.config.ts new file mode 100644 index 000000000..163b6af13 --- /dev/null +++ b/opensearch/rsbuild.config.ts @@ -0,0 +1,48 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { pluginReact } from '@rsbuild/plugin-react'; +import { createConfigForPlugin } from '../rsbuild.shared'; + +export default createConfigForPlugin({ + name: 'OpenSearch', + rsbuildConfig: { + server: { port: 3135 }, + dev: { hmr: false, liveReload: false }, + plugins: [pluginReact()], + }, + moduleFederation: { + exposes: { + './OpenSearchDatasource': './src/datasources/opensearch-datasource', + './OpenSearchLogQuery': './src/queries/opensearch-log-query', + }, + shared: { + react: { requiredVersion: '18.2.0', singleton: true }, + 'react-dom': { requiredVersion: '18.2.0', singleton: true }, + echarts: { requiredVersion: '5.5.0', singleton: true }, + 'date-fns': { singleton: true }, + 'date-fns-tz': { requiredVersion: '^3.2.0', version: '3.2.0', singleton: true }, + lodash: { requiredVersion: '^4.17.21', singleton: true }, + '@perses-dev/components': { singleton: true }, + '@perses-dev/plugin-system': { singleton: true }, + '@perses-dev/explore': { singleton: true }, + '@perses-dev/dashboards': { singleton: true }, + '@emotion/react': { requiredVersion: '^11.11.3', singleton: true }, + '@emotion/styled': { requiredVersion: '^11.6.0', singleton: true }, + '@hookform/resolvers': { singleton: true }, + '@tanstack/react-query': { singleton: true }, + 'react-hook-form': { singleton: true }, + 'react-router-dom': { singleton: true }, + }, + }, +}); diff --git a/opensearch/schemas/datasources/opensearch.cue b/opensearch/schemas/datasources/opensearch.cue new file mode 100644 index 000000000..8ca94a88f --- /dev/null +++ b/opensearch/schemas/datasources/opensearch.cue @@ -0,0 +1,28 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "github.com/perses/shared/cue/common" + commonProxy "github.com/perses/shared/cue/common/proxy" +) + +#kind: "OpenSearchDatasource" + +kind: #kind +spec: { + commonProxy.#baseHTTPDatasourceSpec +} + +#selector: common.#datasourceSelector & {_kind: #kind} diff --git a/opensearch/schemas/datasources/tests/valid/opensearch.json b/opensearch/schemas/datasources/tests/valid/opensearch.json new file mode 100644 index 000000000..0be3385a5 --- /dev/null +++ b/opensearch/schemas/datasources/tests/valid/opensearch.json @@ -0,0 +1,6 @@ +{ + "kind": "OpenSearchDatasource", + "spec": { + "directUrl": "http://localhost:9200" + } +} diff --git a/opensearch/schemas/queries/opensearch-log-query/query.cue b/opensearch/schemas/queries/opensearch-log-query/query.cue new file mode 100644 index 000000000..8dcd6005e --- /dev/null +++ b/opensearch/schemas/queries/opensearch-log-query/query.cue @@ -0,0 +1,29 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "strings" + ds "github.com/perses/plugins/opensearch/schemas/datasources:model" +) + +kind: "OpenSearchLogQuery" +spec: close({ + ds.#selector + query: strings.MinRunes(1) + index?: strings.MinRunes(1) + timestampField?: strings.MinRunes(1) + messageField?: strings.MinRunes(1) + disableTimeFilter?: bool +}) diff --git a/opensearch/schemas/queries/opensearch-log-query/tests/invalid/empty-index.json b/opensearch/schemas/queries/opensearch-log-query/tests/invalid/empty-index.json new file mode 100644 index 000000000..41391f35f --- /dev/null +++ b/opensearch/schemas/queries/opensearch-log-query/tests/invalid/empty-index.json @@ -0,0 +1,11 @@ +{ + "kind": "OpenSearchLogQuery", + "spec": { + "datasource": { + "kind": "OpenSearchDatasource", + "name": "opensearch" + }, + "query": "source=logs-*", + "index": "" + } +} diff --git a/opensearch/schemas/queries/opensearch-log-query/tests/invalid/empty-message-field.json b/opensearch/schemas/queries/opensearch-log-query/tests/invalid/empty-message-field.json new file mode 100644 index 000000000..7c917f2b4 --- /dev/null +++ b/opensearch/schemas/queries/opensearch-log-query/tests/invalid/empty-message-field.json @@ -0,0 +1,11 @@ +{ + "kind": "OpenSearchLogQuery", + "spec": { + "datasource": { + "kind": "OpenSearchDatasource", + "name": "opensearch" + }, + "query": "source=logs-*", + "messageField": "" + } +} diff --git a/opensearch/schemas/queries/opensearch-log-query/tests/invalid/empty-query.json b/opensearch/schemas/queries/opensearch-log-query/tests/invalid/empty-query.json new file mode 100644 index 000000000..d1373c4e6 --- /dev/null +++ b/opensearch/schemas/queries/opensearch-log-query/tests/invalid/empty-query.json @@ -0,0 +1,10 @@ +{ + "kind": "OpenSearchLogQuery", + "spec": { + "datasource": { + "kind": "OpenSearchDatasource", + "name": "opensearch" + }, + "query": "" + } +} diff --git a/opensearch/schemas/queries/opensearch-log-query/tests/invalid/empty-timestamp-field.json b/opensearch/schemas/queries/opensearch-log-query/tests/invalid/empty-timestamp-field.json new file mode 100644 index 000000000..ced08adce --- /dev/null +++ b/opensearch/schemas/queries/opensearch-log-query/tests/invalid/empty-timestamp-field.json @@ -0,0 +1,11 @@ +{ + "kind": "OpenSearchLogQuery", + "spec": { + "datasource": { + "kind": "OpenSearchDatasource", + "name": "opensearch" + }, + "query": "source=logs-*", + "timestampField": "" + } +} diff --git a/opensearch/schemas/queries/opensearch-log-query/tests/valid/basic.json b/opensearch/schemas/queries/opensearch-log-query/tests/valid/basic.json new file mode 100644 index 000000000..1b651313c --- /dev/null +++ b/opensearch/schemas/queries/opensearch-log-query/tests/valid/basic.json @@ -0,0 +1,13 @@ +{ + "kind": "OpenSearchLogQuery", + "spec": { + "datasource": { + "kind": "OpenSearchDatasource", + "name": "opensearch" + }, + "query": "source=otel-logs-* | where traceId='abc123'", + "index": "otel-logs-*", + "timestampField": "@timestamp", + "messageField": "body" + } +} diff --git a/opensearch/schemas/queries/opensearch-log-query/tests/valid/disable-time-filter.json b/opensearch/schemas/queries/opensearch-log-query/tests/valid/disable-time-filter.json new file mode 100644 index 000000000..a2987440f --- /dev/null +++ b/opensearch/schemas/queries/opensearch-log-query/tests/valid/disable-time-filter.json @@ -0,0 +1,7 @@ +{ + "kind": "OpenSearchLogQuery", + "spec": { + "query": "source=otel-logs-* | where traceId='abc123'", + "disableTimeFilter": true + } +} diff --git a/opensearch/sdk/go/datasource/datasource.go b/opensearch/sdk/go/datasource/datasource.go new file mode 100644 index 000000000..eb897f714 --- /dev/null +++ b/opensearch/sdk/go/datasource/datasource.go @@ -0,0 +1,109 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datasource + +import ( + "encoding/json" + "fmt" + + "github.com/perses/perses/go-sdk/datasource" + "github.com/perses/perses/pkg/model/api/v1/datasource/http" +) + +const ( + PluginKind = "OpenSearchDatasource" +) + +type PluginSpec struct { + DirectURL string `json:"directUrl,omitempty" yaml:"directUrl,omitempty"` + Proxy *http.Proxy `json:"proxy,omitempty" yaml:"proxy,omitempty"` +} + +func (s *PluginSpec) UnmarshalJSON(data []byte) error { + type plain PluginSpec + var tmp PluginSpec + if err := json.Unmarshal(data, (*plain)(&tmp)); err != nil { + return err + } + if err := (&tmp).validate(); err != nil { + return err + } + *s = tmp + return nil +} + +func (s *PluginSpec) UnmarshalYAML(unmarshal func(interface{}) error) error { + var tmp PluginSpec + type plain PluginSpec + if err := unmarshal((*plain)(&tmp)); err != nil { + return err + } + if err := (&tmp).validate(); err != nil { + return err + } + *s = tmp + return nil +} + +func (s *PluginSpec) validate() error { + if len(s.DirectURL) == 0 && s.Proxy == nil { + return fmt.Errorf("directUrl or proxy cannot be empty") + } + if len(s.DirectURL) > 0 && s.Proxy != nil { + return fmt.Errorf("at most directUrl or proxy must be configured") + } + return nil +} + +type Option func(plugin *Builder) error + +func create(options ...Option) (Builder, error) { + builder := &Builder{ + PluginSpec: PluginSpec{}, + } + + var defaults []Option + + for _, opt := range append(defaults, options...) { + if err := opt(builder); err != nil { + return *builder, err + } + } + + return *builder, nil +} + +type Builder struct { + PluginSpec `json:",inline" yaml:",inline"` +} + +func OpenSearch(options ...Option) datasource.Option { + return func(builder *datasource.Builder) error { + plugin, err := create(options...) + if err != nil { + return err + } + + builder.Spec.Plugin.Kind = PluginKind + builder.Spec.Plugin.Spec = plugin.PluginSpec + return nil + } +} + +func Selector(datasourceName string) *datasource.Selector { + return &datasource.Selector{ + Kind: PluginKind, + Name: datasourceName, + } +} diff --git a/opensearch/sdk/go/datasource/options.go b/opensearch/sdk/go/datasource/options.go new file mode 100644 index 000000000..07f0d2303 --- /dev/null +++ b/opensearch/sdk/go/datasource/options.go @@ -0,0 +1,36 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datasource + +import ( + "github.com/perses/perses/go-sdk/http" +) + +func DirectURL(url string) Option { + return func(builder *Builder) error { + builder.DirectURL = url + return nil + } +} + +func HTTPProxy(url string, options ...http.Option) Option { + return func(builder *Builder) error { + p, err := http.New(url, options...) + if err != nil { + return err + } + builder.Proxy = &p.Proxy + return nil + } +} diff --git a/opensearch/sdk/go/query/log/log.go b/opensearch/sdk/go/query/log/log.go new file mode 100644 index 000000000..057185a62 --- /dev/null +++ b/opensearch/sdk/go/query/log/log.go @@ -0,0 +1,104 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package log + +import ( + "encoding/json" + "fmt" + + "github.com/perses/perses/go-sdk/datasource" + "github.com/perses/perses/go-sdk/query" + "github.com/perses/perses/pkg/model/api/v1/common" + "github.com/perses/perses/pkg/model/api/v1/plugin" +) + +const PluginKind = "OpenSearchLogQuery" + +type PluginSpec struct { + Datasource *datasource.Selector `json:"datasource,omitempty" yaml:"datasource,omitempty"` + Query string `json:"query" yaml:"query"` + Index string `json:"index,omitempty" yaml:"index,omitempty"` + TimestampField string `json:"timestampField,omitempty" yaml:"timestampField,omitempty"` + MessageField string `json:"messageField,omitempty" yaml:"messageField,omitempty"` + DisableTimeFilter bool `json:"disableTimeFilter,omitempty" yaml:"disableTimeFilter,omitempty"` +} + +func (s *PluginSpec) UnmarshalJSON(data []byte) error { + type plain PluginSpec + var tmp PluginSpec + if err := json.Unmarshal(data, (*plain)(&tmp)); err != nil { + return err + } + if err := (&tmp).validate(); err != nil { + return err + } + *s = tmp + return nil +} + +func (s *PluginSpec) UnmarshalYAML(unmarshal func(interface{}) error) error { + var tmp PluginSpec + type plain PluginSpec + if err := unmarshal((*plain)(&tmp)); err != nil { + return err + } + if err := (&tmp).validate(); err != nil { + return err + } + *s = tmp + return nil +} + +func (s *PluginSpec) validate() error { + if len(s.Query) == 0 { + return fmt.Errorf("query cannot be empty") + } + return nil +} + +type Option func(plugin *Builder) error + +func create(query string, options ...Option) (Builder, error) { + builder := &Builder{ + PluginSpec: PluginSpec{}, + } + + defaults := []Option{ + Query(query), + } + + for _, opt := range append(defaults, options...) { + if err := opt(builder); err != nil { + return *builder, err + } + } + + return *builder, nil +} + +type Builder struct { + PluginSpec `json:",inline" yaml:",inline"` +} + +func OpenSearchLogQuery(expr string, options ...Option) query.Option { + plg, err := create(expr, options...) + return query.Option{ + Kind: plugin.KindLogQuery, + Plugin: common.Plugin{ + Kind: PluginKind, + Spec: plg, + }, + Error: err, + } +} diff --git a/opensearch/sdk/go/query/log/log_test.go b/opensearch/sdk/go/query/log/log_test.go new file mode 100644 index 000000000..ef761d76e --- /dev/null +++ b/opensearch/sdk/go/query/log/log_test.go @@ -0,0 +1,105 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package log + +import ( + "encoding/json" + "testing" +) + +func TestOpenSearchLogQueryBuilder(t *testing.T) { + q := OpenSearchLogQuery( + "source=logs-* | where traceId='$traceId'", + Datasource("my-os"), + Index("logs-*"), + TimestampField("time"), + MessageField("body"), + ) + if q.Error != nil { + t.Fatalf("unexpected error: %v", q.Error) + } + if q.Plugin.Kind != "OpenSearchLogQuery" { + t.Fatalf("unexpected kind: %s", q.Plugin.Kind) + } + + raw, err := json.Marshal(q.Plugin.Spec) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + if out["query"] != "source=logs-* | where traceId='$traceId'" { + t.Errorf("query mismatch: %v", out["query"]) + } + if out["index"] != "logs-*" { + t.Errorf("index mismatch: %v", out["index"]) + } + if out["timestampField"] != "time" { + t.Errorf("timestampField mismatch: %v", out["timestampField"]) + } + if out["messageField"] != "body" { + t.Errorf("messageField mismatch: %v", out["messageField"]) + } +} + +func TestOpenSearchLogQueryOmitsEmptyOptionalFields(t *testing.T) { + q := OpenSearchLogQuery("source=logs-*") + raw, _ := json.Marshal(q.Plugin.Spec) + var out map[string]any + _ = json.Unmarshal(raw, &out) + + for _, f := range []string{"index", "timestampField", "messageField", "datasource", "disableTimeFilter"} { + if _, present := out[f]; present { + t.Errorf("expected %s to be omitted, got: %v", f, out[f]) + } + } +} + +func TestPluginSpecRejectsEmptyQueryOnUnmarshal(t *testing.T) { + raw := []byte(`{"query":""}`) + var spec PluginSpec + if err := json.Unmarshal(raw, &spec); err == nil { + t.Fatalf("expected error unmarshalling spec with empty query, got nil") + } +} + +func TestPluginSpecAcceptsNonEmptyQueryOnUnmarshal(t *testing.T) { + raw := []byte(`{"query":"source=logs-*"}`) + var spec PluginSpec + if err := json.Unmarshal(raw, &spec); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if spec.Query != "source=logs-*" { + t.Errorf("query mismatch: %q", spec.Query) + } +} + +func TestOpenSearchLogQueryDisableTimeFilter(t *testing.T) { + q := OpenSearchLogQuery("source=logs-*", DisableTimeFilter(true)) + raw, err := json.Marshal(q.Plugin.Spec) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if out["disableTimeFilter"] != true { + t.Errorf("disableTimeFilter mismatch: %v", out["disableTimeFilter"]) + } +} diff --git a/opensearch/sdk/go/query/log/options.go b/opensearch/sdk/go/query/log/options.go new file mode 100644 index 000000000..e76c0da1c --- /dev/null +++ b/opensearch/sdk/go/query/log/options.go @@ -0,0 +1,60 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package log + +import ( + opensearchDatasource "github.com/perses/plugins/opensearch/sdk/go/datasource" +) + +func Query(expr string) Option { + return func(builder *Builder) error { + builder.Query = expr + return nil + } +} + +func Datasource(datasourceName string) Option { + return func(builder *Builder) error { + builder.Datasource = opensearchDatasource.Selector(datasourceName) + return nil + } +} + +func Index(index string) Option { + return func(builder *Builder) error { + builder.Index = index + return nil + } +} + +func TimestampField(field string) Option { + return func(builder *Builder) error { + builder.TimestampField = field + return nil + } +} + +func MessageField(field string) Option { + return func(builder *Builder) error { + builder.MessageField = field + return nil + } +} + +func DisableTimeFilter(disable bool) Option { + return func(builder *Builder) error { + builder.DisableTimeFilter = disable + return nil + } +} diff --git a/opensearch/src/bootstrap.tsx b/opensearch/src/bootstrap.tsx new file mode 100644 index 000000000..b08e3e695 --- /dev/null +++ b/opensearch/src/bootstrap.tsx @@ -0,0 +1,18 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import React from 'react'; +import ReactDOM from 'react-dom/client'; + +const root = ReactDOM.createRoot(document.getElementById('root') as HTMLElement); +root.render(); diff --git a/opensearch/src/datasources/index.ts b/opensearch/src/datasources/index.ts new file mode 100644 index 000000000..dcd27f535 --- /dev/null +++ b/opensearch/src/datasources/index.ts @@ -0,0 +1,14 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export * from './opensearch-datasource'; diff --git a/opensearch/src/datasources/opensearch-datasource/OpenSearchDatasource.tsx b/opensearch/src/datasources/opensearch-datasource/OpenSearchDatasource.tsx new file mode 100644 index 000000000..2c25d7f23 --- /dev/null +++ b/opensearch/src/datasources/opensearch-datasource/OpenSearchDatasource.tsx @@ -0,0 +1,42 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { DatasourcePlugin } from '@perses-dev/plugin-system'; +import { OpenSearchClient, ppl } from '../../model/opensearch-client'; +import { OpenSearchDatasourceSpec } from './opensearch-datasource-types'; +import { OpenSearchDatasourceEditor } from './OpenSearchDatasourceEditor'; + +const createClient: DatasourcePlugin['createClient'] = (spec, options) => { + const { directUrl, proxy } = spec; + const { proxyUrl } = options; + + const datasourceUrl = directUrl ?? proxyUrl; + if (datasourceUrl === undefined) { + throw new Error('No URL specified for OpenSearch client. You can use directUrl in the spec to configure it.'); + } + + const specHeaders = proxy?.spec.headers; + + return { + options: { + datasourceUrl, + }, + ppl: (params, headers) => ppl(params, { datasourceUrl, headers: headers ?? specHeaders }), + }; +}; + +export const OpenSearchDatasource: DatasourcePlugin = { + createClient, + OptionsEditorComponent: OpenSearchDatasourceEditor, + createInitialOptions: () => ({ directUrl: '' }), +}; diff --git a/opensearch/src/datasources/opensearch-datasource/OpenSearchDatasourceEditor.tsx b/opensearch/src/datasources/opensearch-datasource/OpenSearchDatasourceEditor.tsx new file mode 100644 index 000000000..8d175c43c --- /dev/null +++ b/opensearch/src/datasources/opensearch-datasource/OpenSearchDatasourceEditor.tsx @@ -0,0 +1,55 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { HTTPSettingsEditor } from '@perses-dev/plugin-system'; +import { ReactElement } from 'react'; +import { OpenSearchDatasourceSpec } from './opensearch-datasource-types'; + +export interface OpenSearchDatasourceEditorProps { + value: OpenSearchDatasourceSpec; + onChange: (next: OpenSearchDatasourceSpec) => void; + isReadonly?: boolean; +} + +export function OpenSearchDatasourceEditor(props: OpenSearchDatasourceEditorProps): ReactElement { + const { value, onChange, isReadonly } = props; + + const initialSpecDirect: OpenSearchDatasourceSpec = { + directUrl: '', + }; + + const initialSpecProxy: OpenSearchDatasourceSpec = { + proxy: { + kind: 'HTTPProxy', + spec: { + allowedEndpoints: [ + { + endpointPattern: '/_plugins/_ppl', + method: 'POST', + }, + ], + url: '', + }, + }, + }; + + return ( + + ); +} diff --git a/opensearch/src/datasources/opensearch-datasource/index.ts b/opensearch/src/datasources/opensearch-datasource/index.ts new file mode 100644 index 000000000..ceedf0736 --- /dev/null +++ b/opensearch/src/datasources/opensearch-datasource/index.ts @@ -0,0 +1,16 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export * from './OpenSearchDatasource'; +export * from './OpenSearchDatasourceEditor'; +export * from './opensearch-datasource-types'; diff --git a/opensearch/src/datasources/opensearch-datasource/opensearch-datasource-types.ts b/opensearch/src/datasources/opensearch-datasource/opensearch-datasource-types.ts new file mode 100644 index 000000000..d3a588fcb --- /dev/null +++ b/opensearch/src/datasources/opensearch-datasource/opensearch-datasource-types.ts @@ -0,0 +1,19 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { HTTPProxy } from '@perses-dev/spec'; + +export interface OpenSearchDatasourceSpec { + directUrl?: string; + proxy?: HTTPProxy; +} diff --git a/opensearch/src/env.d.ts b/opensearch/src/env.d.ts new file mode 100644 index 000000000..e216e17b1 --- /dev/null +++ b/opensearch/src/env.d.ts @@ -0,0 +1,14 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// diff --git a/opensearch/src/getPluginModule.ts b/opensearch/src/getPluginModule.ts new file mode 100644 index 000000000..ee2f25a74 --- /dev/null +++ b/opensearch/src/getPluginModule.ts @@ -0,0 +1,30 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { PluginModuleResource, PluginModuleSpec } from '@perses-dev/plugin-system'; +import packageJson from '../package.json'; + +/** + * Returns the plugin module information from package.json + */ +export function getPluginModule(): PluginModuleResource { + const { name, version, perses } = packageJson; + return { + kind: 'PluginModule', + metadata: { + name, + version, + }, + spec: perses as PluginModuleSpec, + }; +} diff --git a/opensearch/src/index-federation.ts b/opensearch/src/index-federation.ts new file mode 100644 index 000000000..36f748007 --- /dev/null +++ b/opensearch/src/index-federation.ts @@ -0,0 +1,14 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import('./bootstrap'); diff --git a/opensearch/src/index.ts b/opensearch/src/index.ts new file mode 100644 index 000000000..505ad1e88 --- /dev/null +++ b/opensearch/src/index.ts @@ -0,0 +1,17 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export { getPluginModule } from './getPluginModule'; +export * from './model'; +export * from './queries'; +export * from './datasources'; diff --git a/opensearch/src/model/index.ts b/opensearch/src/model/index.ts new file mode 100644 index 000000000..4e141784e --- /dev/null +++ b/opensearch/src/model/index.ts @@ -0,0 +1,16 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export * from './opensearch-client'; +export * from './opensearch-client-types'; +export * from './opensearch-selectors'; diff --git a/opensearch/src/model/opensearch-client-types.ts b/opensearch/src/model/opensearch-client-types.ts new file mode 100644 index 000000000..81fd2de25 --- /dev/null +++ b/opensearch/src/model/opensearch-client-types.ts @@ -0,0 +1,35 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export type OpenSearchRequestHeaders = Record; + +export interface OpenSearchPPLColumn { + name: string; + type: string; +} + +export interface OpenSearchPPLResponse { + schema: OpenSearchPPLColumn[]; + datarows: Array>; + total?: number; + size?: number; +} + +export interface OpenSearchErrorResponse { + error?: { + type?: string; + reason?: string; + details?: string; + }; + status?: number; +} diff --git a/opensearch/src/model/opensearch-client.test.ts b/opensearch/src/model/opensearch-client.test.ts new file mode 100644 index 000000000..1119839ce --- /dev/null +++ b/opensearch/src/model/opensearch-client.test.ts @@ -0,0 +1,125 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { ppl, OpenSearchPPLError } from './opensearch-client'; + +describe('opensearch-client', () => { + const originalFetch = global.fetch; + + afterEach(() => { + global.fetch = originalFetch; + jest.resetAllMocks(); + }); + + function mockFetch(response: { ok: boolean; status: number; body: unknown }): jest.Mock { + const mock = jest.fn(async () => ({ + ok: response.ok, + status: response.status, + text: async (): Promise => + typeof response.body === 'string' ? response.body : JSON.stringify(response.body), + json: async (): Promise => response.body, + })) as unknown as jest.Mock; + global.fetch = mock as unknown as typeof fetch; + return mock; + } + + it('POSTs to /_plugins/_ppl with the right body and headers', async () => { + const mock = mockFetch({ + ok: true, + status: 200, + body: { schema: [], datarows: [] }, + }); + + await ppl({ query: 'source=logs-*' }, { datasourceUrl: 'http://localhost:9200' }); + + expect(mock).toHaveBeenCalledTimes(1); + const [url, init] = mock.mock.calls[0] as [string, RequestInit]; + expect(url).toBe('http://localhost:9200/_plugins/_ppl'); + expect(init.method).toBe('POST'); + expect((init.headers as Record)['Content-Type']).toBe('application/json'); + expect(init.body).toBe(JSON.stringify({ query: 'source=logs-*' })); + }); + + it('forwards extra headers from options', async () => { + const mock = mockFetch({ ok: true, status: 200, body: { schema: [], datarows: [] } }); + await ppl( + { query: 'source=logs-*' }, + { datasourceUrl: 'http://localhost:9200', headers: { Authorization: 'Basic xyz' } } + ); + const [, init] = mock.mock.calls[0] as [string, RequestInit]; + expect((init.headers as Record).Authorization).toBe('Basic xyz'); + }); + + it('throws OpenSearchPPLError carrying status and body on non-200', async () => { + mockFetch({ ok: false, status: 400, body: '{"error":{"reason":"bad PPL"}}' }); + await expect(ppl({ query: 'broken' }, { datasourceUrl: 'http://localhost:9200' })).rejects.toMatchObject({ + name: 'OpenSearchPPLError', + status: 400, + body: '{"error":{"reason":"bad PPL"}}', + }); + await expect(ppl({ query: 'broken' }, { datasourceUrl: 'http://localhost:9200' })).rejects.toBeInstanceOf( + OpenSearchPPLError + ); + }); + + it('builds a URL relative to window.location.origin when datasourceUrl is a path', async () => { + const mock = mockFetch({ ok: true, status: 200, body: { schema: [], datarows: [] } }); + await ppl({ query: 'q' }, { datasourceUrl: '/api/datasources/proxy/1/' }); + const [url] = mock.mock.calls[0] as [string]; + expect(url).toContain('/api/datasources/proxy/1/_plugins/_ppl'); + }); + + describe('OpenSearchPPLError.message', () => { + it('uses error.reason from a JSON body when present', () => { + const err = new OpenSearchPPLError(400, '{"error":{"reason":"bad PPL","type":"SyntaxCheckException"}}'); + expect(err.message).toBe('OpenSearch PPL request failed (400): bad PPL'); + expect(err.body).toBe('{"error":{"reason":"bad PPL","type":"SyntaxCheckException"}}'); + }); + + it('falls back to error.details when reason is absent', () => { + const err = new OpenSearchPPLError(400, '{"error":{"details":"timestamp:2026-... in unsupported format"}}'); + expect(err.message).toBe('OpenSearch PPL request failed (400): timestamp:2026-... in unsupported format'); + }); + + it('omits the body entirely when the body is not JSON', () => { + const err = new OpenSearchPPLError(502, 'Bad Gateway'); + expect(err.message).toBe('OpenSearch PPL request failed (502)'); + expect(err.body).toBe('Bad Gateway'); + }); + + it('omits the body when JSON has no error field', () => { + const err = new OpenSearchPPLError(500, '{"status":"weird"}'); + expect(err.message).toBe('OpenSearch PPL request failed (500)'); + }); + + it('combines reason and details when both are present and differ', () => { + const err = new OpenSearchPPLError( + 400, + '{"error":{"reason":"Invalid Query","details":"can\'t resolve Symbol(name=@timestamp)"}}' + ); + expect(err.message).toBe( + "OpenSearch PPL request failed (400): Invalid Query — can't resolve Symbol(name=@timestamp)" + ); + }); + + it('does not duplicate text when reason equals details', () => { + const err = new OpenSearchPPLError(400, '{"error":{"reason":"same text","details":"same text"}}'); + expect(err.message).toBe('OpenSearch PPL request failed (400): same text'); + }); + + it('uses reason alone when details is an empty string', () => { + const err = new OpenSearchPPLError(400, '{"error":{"reason":"Invalid Query","details":""}}'); + expect(err.message).toBe('OpenSearch PPL request failed (400): Invalid Query'); + }); + }); +}); diff --git a/opensearch/src/model/opensearch-client.ts b/opensearch/src/model/opensearch-client.ts new file mode 100644 index 000000000..26c7dd8c3 --- /dev/null +++ b/opensearch/src/model/opensearch-client.ts @@ -0,0 +1,96 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { OpenSearchPPLResponse, OpenSearchRequestHeaders } from './opensearch-client-types'; + +export interface OpenSearchPPLParams { + query: string; +} + +export interface OpenSearchApiOptions { + datasourceUrl: string; + headers?: OpenSearchRequestHeaders; +} + +export interface OpenSearchClient { + options: { + datasourceUrl: string; + }; + ppl: (params: OpenSearchPPLParams, headers?: OpenSearchRequestHeaders) => Promise; +} + +export class OpenSearchPPLError extends Error { + constructor( + public readonly status: number, + public readonly body: string, + message?: string + ) { + super(message ?? buildShortMessage(status, body)); + this.name = 'OpenSearchPPLError'; + } +} + +function buildShortMessage(status: number, body: string): string { + try { + const parsed = JSON.parse(body) as { error?: { reason?: string; details?: string } }; + const reason = parsed?.error?.reason; + const details = parsed?.error?.details; + // OpenSearch often puts a generic phrase in `reason` ("Invalid Query") and the + // actionable text in `details` ("can't resolve Symbol(name=@timestamp)"). Surface + // both so the user can diagnose the failure. + const suffix = reason && details && reason !== details ? `${reason} — ${details}` : (reason ?? details); + if (suffix) { + return `OpenSearch PPL request failed (${status}): ${suffix}`; + } + } catch { + // body wasn't JSON — fall through to the bare-status form + } + return `OpenSearch PPL request failed (${status})`; +} + +function buildUrl(path: string, datasourceUrl: string): URL { + if (datasourceUrl.startsWith('http://') || datasourceUrl.startsWith('https://')) { + return new URL(path, datasourceUrl); + } + + let fullPath: string; + if (datasourceUrl.endsWith('/') && path.startsWith('/')) { + fullPath = datasourceUrl + path.slice(1); + } else if (!datasourceUrl.endsWith('/') && !path.startsWith('/')) { + fullPath = datasourceUrl + '/' + path; + } else { + fullPath = datasourceUrl + path; + } + + return new URL(fullPath, window.location.origin); +} + +export async function ppl(params: OpenSearchPPLParams, options: OpenSearchApiOptions): Promise { + const url = buildUrl('/_plugins/_ppl', options.datasourceUrl); + + const response = await fetch(url.toString(), { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...options.headers, + }, + body: JSON.stringify({ query: params.query }), + }); + + if (!response.ok) { + const body = await response.text(); + throw new OpenSearchPPLError(response.status, body); + } + + return response.json(); +} diff --git a/opensearch/src/model/opensearch-selectors.ts b/opensearch/src/model/opensearch-selectors.ts new file mode 100644 index 000000000..82ad20907 --- /dev/null +++ b/opensearch/src/model/opensearch-selectors.ts @@ -0,0 +1,33 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { DatasourceSelector } from '@perses-dev/spec'; +import { DatasourceSelectValue, isVariableDatasource } from '@perses-dev/plugin-system'; + +export const OPENSEARCH_DATASOURCE_KIND = 'OpenSearchDatasource' as const; + +export interface OpenSearchDatasourceSelector extends DatasourceSelector { + kind: typeof OPENSEARCH_DATASOURCE_KIND; +} + +export const DEFAULT_OPENSEARCH: OpenSearchDatasourceSelector = { kind: OPENSEARCH_DATASOURCE_KIND }; + +export function isDefaultOpenSearchSelector(datasourceSelectValue: DatasourceSelectValue): boolean { + return !isVariableDatasource(datasourceSelectValue) && datasourceSelectValue.name === undefined; +} + +export function isOpenSearchDatasourceSelector( + datasourceSelectValue: DatasourceSelectValue +): datasourceSelectValue is OpenSearchDatasourceSelector { + return isVariableDatasource(datasourceSelectValue) || datasourceSelectValue.kind === OPENSEARCH_DATASOURCE_KIND; +} diff --git a/opensearch/src/queries/constants.ts b/opensearch/src/queries/constants.ts new file mode 100644 index 000000000..be59e5dbe --- /dev/null +++ b/opensearch/src/queries/constants.ts @@ -0,0 +1,29 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export const DATASOURCE_KIND = 'OpenSearchDatasource'; +export const DEFAULT_DATASOURCE = { kind: DATASOURCE_KIND }; + +// Default field names inside OpenSearch log documents. OpenSearch doesn't enforce a schema, +// so these are used as reasonable fallbacks when mapping rows to log entries. +export const DEFAULT_TIMESTAMP_FIELDS = ['@timestamp', 'timestamp', 'time']; +export const DEFAULT_MESSAGE_FIELDS = ['message', 'log', 'body']; + +// OpenSearch PPL reference documentation. +export const PPL_DOCS_URL = 'https://opensearch.org/docs/latest/search-plugins/sql/ppl/index/'; + +// A handful of copy-pasteable PPL examples shown in the editor's "Query Examples" disclosure. +export const PPL_QUERY_EXAMPLES = `source=logs-* | where level="error" | head 20 +source=logs-* | fields @timestamp, service, message | head 50 +source=logs-* | stats count() by service +source=logs-* | where traceId='$traceId'`; diff --git a/opensearch/src/queries/index.ts b/opensearch/src/queries/index.ts new file mode 100644 index 000000000..ba9f3c679 --- /dev/null +++ b/opensearch/src/queries/index.ts @@ -0,0 +1,14 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export * from './opensearch-log-query'; diff --git a/opensearch/src/queries/opensearch-log-query/OpenSearchLogQuery.tsx b/opensearch/src/queries/opensearch-log-query/OpenSearchLogQuery.tsx new file mode 100644 index 000000000..d8ba93fd7 --- /dev/null +++ b/opensearch/src/queries/opensearch-log-query/OpenSearchLogQuery.tsx @@ -0,0 +1,32 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { parseVariables } from '@perses-dev/plugin-system'; +import { getOpenSearchLogData } from './get-opensearch-log-data'; +import { OpenSearchLogQueryEditor } from './OpenSearchLogQueryEditor'; +import { OpenSearchLogQuerySpec } from './opensearch-log-query-types'; +import { LogQueryPlugin } from './log-query-plugin-interface'; + +export const OpenSearchLogQuery: LogQueryPlugin = { + getLogData: getOpenSearchLogData, + OptionsEditorComponent: OpenSearchLogQueryEditor, + createInitialOptions: () => ({ query: '' }), + dependsOn: (spec) => { + const queryVariables = parseVariables(spec.query); + const indexVariables = spec.index ? parseVariables(spec.index) : []; + const allVariables = [...new Set([...queryVariables, ...indexVariables])]; + return { + variables: allVariables, + }; + }, +}; diff --git a/opensearch/src/queries/opensearch-log-query/OpenSearchLogQueryEditor.test.tsx b/opensearch/src/queries/opensearch-log-query/OpenSearchLogQueryEditor.test.tsx new file mode 100644 index 000000000..298d9b379 --- /dev/null +++ b/opensearch/src/queries/opensearch-log-query/OpenSearchLogQueryEditor.test.tsx @@ -0,0 +1,93 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { render, screen, fireEvent } from '@testing-library/react'; +import { PPL_DOCS_URL } from '../constants'; +import { OpenSearchLogQueryEditor } from './OpenSearchLogQueryEditor'; +import { OpenSearchLogQuerySpec } from './opensearch-log-query-types'; + +jest.mock('@perses-dev/plugin-system', () => ({ + ...jest.requireActual('@perses-dev/plugin-system'), + DatasourceSelect: (): JSX.Element =>
, + useDatasourceSelectValueToSelector: (): undefined => undefined, + isVariableDatasource: (): boolean => false, +})); + +// Make the Mod+Enter handler call its callback on any keydown, decoupling the test +// from the host lib's platform-specific key detection. +jest.mock('@perses-dev/dashboards', () => ({ + createModEnterHandler: (cb: () => void) => (): void => cb(), +})); + +function setup(initial: OpenSearchLogQuerySpec = { query: '' }): { onChange: jest.Mock } { + const onChange = jest.fn(); + render(); + return { onChange }; +} + +describe('OpenSearchLogQueryEditor', () => { + it('renders datasource picker, index, timestamp/message field, and query inputs', () => { + setup(); + expect(screen.getByTestId('datasource-select')).toBeInTheDocument(); + expect(screen.getByPlaceholderText('e.g. logs-*')).toBeInTheDocument(); + expect(screen.getByPlaceholderText('@timestamp')).toBeInTheDocument(); + expect(screen.getByPlaceholderText('message')).toBeInTheDocument(); + expect(screen.getByPlaceholderText(/source=logs-/)).toBeInTheDocument(); + }); + + it('persists timestampField into spec when typed', () => { + const { onChange } = setup(); + fireEvent.change(screen.getByPlaceholderText('@timestamp'), { target: { value: 'time' } }); + expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ timestampField: 'time' })); + }); + + it('clears messageField when emptied', () => { + const { onChange } = setup({ query: '', messageField: 'body' }); + fireEvent.change(screen.getByPlaceholderText('message'), { target: { value: '' } }); + expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ messageField: undefined })); + }); + + it('toggles disableTimeFilter on when the checkbox is clicked', () => { + const { onChange } = setup(); + fireEvent.click(screen.getByLabelText('Disable automatic time filtering')); + expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ disableTimeFilter: true })); + }); + + it('clears disableTimeFilter when unchecked', () => { + const { onChange } = setup({ query: '', disableTimeFilter: true }); + fireEvent.click(screen.getByLabelText('Disable automatic time filtering')); + expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ disableTimeFilter: undefined })); + }); + + it('notes that source= in the PPL query overrides the Index field', () => { + setup(); + expect(screen.getByText(/Ignored when the PPL query starts with/i)).toBeInTheDocument(); + }); + + it('links to the OpenSearch PPL documentation', () => { + setup(); + const link = screen.getByRole('link', { name: /PPL/i }); + expect(link).toHaveAttribute('href', PPL_DOCS_URL); + }); + + it('renders a Query Examples disclosure', () => { + setup(); + expect(screen.getByText('Query Examples')).toBeInTheDocument(); + }); + + it('commits the query on Mod+Enter', () => { + const { onChange } = setup({ query: 'source=logs-*' }); + fireEvent.keyDown(screen.getByPlaceholderText(/source=logs-/), { key: 'Enter', ctrlKey: true }); + expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ query: 'source=logs-*' })); + }); +}); diff --git a/opensearch/src/queries/opensearch-log-query/OpenSearchLogQueryEditor.tsx b/opensearch/src/queries/opensearch-log-query/OpenSearchLogQueryEditor.tsx new file mode 100644 index 000000000..d1a4355cd --- /dev/null +++ b/opensearch/src/queries/opensearch-log-query/OpenSearchLogQueryEditor.tsx @@ -0,0 +1,193 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { + DatasourceSelect, + DatasourceSelectProps, + isVariableDatasource, + OptionsEditorProps, + useDatasourceSelectValueToSelector, +} from '@perses-dev/plugin-system'; +import { Checkbox, FormControlLabel, InputLabel, Link, Stack, TextField, Typography } from '@mui/material'; +import { ReactElement } from 'react'; +import { produce } from 'immer'; +import { createModEnterHandler } from '@perses-dev/dashboards'; +import { isDefaultOpenSearchSelector, OPENSEARCH_DATASOURCE_KIND, OpenSearchDatasourceSelector } from '../../model'; +import { DATASOURCE_KIND, DEFAULT_DATASOURCE, PPL_DOCS_URL, PPL_QUERY_EXAMPLES } from '../constants'; +import { useQueryState } from '../query-editor-model'; +import { OpenSearchLogQuerySpec } from './opensearch-log-query-types'; + +type OpenSearchQueryEditorProps = OptionsEditorProps; + +const examplesStyle: React.CSSProperties = { + fontSize: '11px', + color: '#777', + backgroundColor: '#f5f5f5', + padding: '8px', + borderRadius: '4px', + fontFamily: 'Monaco, Menlo, "Ubuntu Mono", monospace', + whiteSpace: 'pre-wrap', + lineHeight: '1.3', +}; + +export function OpenSearchLogQueryEditor(props: OpenSearchQueryEditorProps): ReactElement { + const { onChange, value } = props; + const { datasource } = value; + const datasourceSelectValue = datasource ?? DEFAULT_DATASOURCE; + const selectedDatasource = useDatasourceSelectValueToSelector( + datasourceSelectValue, + OPENSEARCH_DATASOURCE_KIND + ) as OpenSearchDatasourceSelector; + + const { query, handleQueryChange, handleQueryBlur } = useQueryState(props); + + const handleDatasourceChange: DatasourceSelectProps['onChange'] = (newDatasourceSelection) => { + if (!isVariableDatasource(newDatasourceSelection) && newDatasourceSelection.kind === DATASOURCE_KIND) { + onChange( + produce(value, (draft) => { + draft.datasource = isDefaultOpenSearchSelector(newDatasourceSelection) ? undefined : newDatasourceSelection; + }) + ); + return; + } + throw new Error('Got unexpected non OpenSearch datasource selection'); + }; + + const handleIndexChange = (e: React.ChangeEvent): void => { + const next = e.target.value; + onChange( + produce(value, (draft) => { + draft.index = next.length > 0 ? next : undefined; + }) + ); + }; + + const handleStringFieldChange = + (field: 'timestampField' | 'messageField') => + (e: React.ChangeEvent): void => { + const next = e.target.value; + onChange( + produce(value, (draft) => { + draft[field] = next.length > 0 ? next : undefined; + }) + ); + }; + + const handleDisableTimeFilterChange = (e: React.ChangeEvent): void => { + const checked = e.target.checked; + onChange( + produce(value, (draft) => { + draft.disableTimeFilter = checked ? true : undefined; + }) + ); + }; + + // Commit the local query immediately (Mod+Enter), matching the loki/clickhouse editors. + const handleQueryExecute = (): void => { + onChange( + produce(value, (draft) => { + draft.query = query; + }) + ); + }; + + return ( + +
+ Datasource + +
+ +
+ Index pattern + +
+ +
+ + Timestamp field (optional) + + +
+ +
+ + Message field (optional) + + +
+ + + } + label="Disable automatic time filtering" + /> + +
+ PPL Query + handleQueryChange(e.target.value)} + onBlur={handleQueryBlur} + onKeyDown={createModEnterHandler(handleQueryExecute)} + placeholder='e.g. source=logs-* | where level="error"' + inputProps={{ style: { fontFamily: 'monospace' } }} + /> + + Uses OpenSearch{' '} + + PPL + + . Requires the PPL plugin enabled on the OpenSearch cluster. + +
+ +
+ + Query Examples + +
{PPL_QUERY_EXAMPLES}
+
+ + ); +} diff --git a/opensearch/src/queries/opensearch-log-query/get-opensearch-log-data.test.ts b/opensearch/src/queries/opensearch-log-query/get-opensearch-log-data.test.ts new file mode 100644 index 000000000..d15686f8c --- /dev/null +++ b/opensearch/src/queries/opensearch-log-query/get-opensearch-log-data.test.ts @@ -0,0 +1,345 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { OpenSearchDatasource } from '../../datasources/opensearch-datasource'; +import { OpenSearchDatasourceSpec } from '../../datasources/opensearch-datasource/opensearch-datasource-types'; +import { OpenSearchPPLResponse } from '../../model/opensearch-client-types'; +import { OpenSearchLogQuery } from './OpenSearchLogQuery'; +import { LogQueryContext } from './log-query-plugin-interface'; +import { buildBoundedPPL, convertPPLToLogs } from './get-opensearch-log-data'; + +const datasource: OpenSearchDatasourceSpec = { + directUrl: '/test', +}; + +const stubClient = OpenSearchDatasource.createClient(datasource, {}); + +stubClient.ppl = jest.fn(async () => { + const response: OpenSearchPPLResponse = { + schema: [ + { name: '@timestamp', type: 'timestamp' }, + { name: 'message', type: 'text' }, + { name: 'level', type: 'keyword' }, + ], + datarows: [ + ['2025-01-01T00:00:00.000Z', 'Error processing request', 'error'], + ['2025-01-01T00:00:01.000Z', 'Retrying', 'warn'], + ], + total: 2, + size: 2, + }; + return response; +}); + +const getDatasourceClient: jest.Mock = jest.fn(() => stubClient); + +function createStubContext(): LogQueryContext { + return { + datasourceStore: { + getDatasource: jest.fn(), + getDatasourceClient, + listDatasourceSelectItems: jest.fn(), + getLocalDatasources: jest.fn(), + setLocalDatasources: jest.fn(), + getSavedDatasources: jest.fn(), + setSavedDatasources: jest.fn(), + }, + timeRange: { + start: new Date('2025-01-01T00:00:00.000Z'), + end: new Date('2025-01-01T01:00:00.000Z'), + }, + variableState: {}, + }; +} + +describe('OpenSearchLogQuery', () => { + afterEach(() => { + (stubClient.ppl as jest.Mock).mockClear(); + }); + + it('creates initial options with empty query', () => { + expect(OpenSearchLogQuery.createInitialOptions()).toEqual({ query: '' }); + }); + + it('resolves variable dependencies from query and index', () => { + if (!OpenSearchLogQuery.dependsOn) throw new Error('dependsOn is not defined'); + const { variables } = OpenSearchLogQuery.dependsOn( + { + query: 'source=$index | where service="$service" and level="$level"', + index: '$index', + }, + createStubContext() + ); + expect(variables?.sort()).toEqual(['index', 'level', 'service']); + }); + + it('returns empty log data for an empty query without calling the client', async () => { + const result = await OpenSearchLogQuery.getLogData({ query: '' }, createStubContext()); + expect(result.logs.entries).toEqual([]); + expect(result.logs.totalCount).toBe(0); + expect(stubClient.ppl).not.toHaveBeenCalled(); + }); + + it('reports traceId in dependsOn when query references $traceId', () => { + if (!OpenSearchLogQuery.dependsOn) throw new Error('dependsOn is not defined'); + const { variables } = OpenSearchLogQuery.dependsOn( + { query: "source=logs-* | where traceId='$traceId'" }, + createStubContext() + ); + expect(variables).toContain('traceId'); + }); + + it('passes timestampField + messageField through to the bounder and mapper', async () => { + const pplMock = stubClient.ppl as jest.Mock; + pplMock.mockImplementationOnce(async () => ({ + schema: [ + { name: 'time', type: 'timestamp' }, + { name: 'body', type: 'text' }, + ], + datarows: [['2025-01-01T00:00:30.000Z', 'OTel-style row']], + })); + + const result = await OpenSearchLogQuery.getLogData( + { + query: 'source=otel-logs-* | head 1', + timestampField: 'time', + messageField: 'body', + }, + createStubContext() + ); + + const sentQuery = pplMock.mock.calls[0]?.[0]?.query as string; + expect(sentQuery).toContain('`time` >='); + expect(result.logs.entries[0]?.line).toBe('OTel-style row'); + }); + + it('substitutes a $traceId variable into the PPL before sending', async () => { + const pplMock = stubClient.ppl as jest.Mock; + pplMock.mockImplementationOnce(async () => ({ schema: [], datarows: [] })); + + await OpenSearchLogQuery.getLogData( + { query: "source=logs-* | where traceId='$traceId'" }, + { + ...createStubContext(), + variableState: { + traceId: { value: 'abc123', loading: false }, + } as unknown as LogQueryContext['variableState'], + } + ); + + const sentQuery = pplMock.mock.calls[0]?.[0]?.query as string; + expect(sentQuery).toContain("where traceId='abc123'"); + }); + + it('executes a PPL query and maps rows to log entries', async () => { + const result = await OpenSearchLogQuery.getLogData( + { query: 'source=logs-* | where level="error"' }, + createStubContext() + ); + + expect(stubClient.ppl).toHaveBeenCalledTimes(1); + expect(result.logs.totalCount).toBe(2); + expect(result.logs.entries[0]).toEqual({ + timestamp: new Date('2025-01-01T00:00:00.000Z').getTime() / 1000, + line: 'Error processing request', + labels: { level: 'error' }, + }); + expect(result.metadata?.executedQueryString).toContain('where level="error"'); + }); +}); + +describe('buildBoundedPPL', () => { + const start = new Date('2025-01-01T00:00:00.000Z'); + const end = new Date('2025-01-01T01:00:00.000Z'); + + it('injects the time-range filter immediately after the source clause', () => { + const q = buildBoundedPPL('source=logs-* | where level="error"', start, end); + expect(q).toBe( + "source=logs-* | where `@timestamp` >= '2025-01-01T00:00:00.000Z' and `@timestamp` <= '2025-01-01T01:00:00.000Z' | where level=\"error\"" + ); + }); + + it('prepends source= when the user query does not declare one', () => { + const q = buildBoundedPPL('where level="error"', start, end, { index: 'logs-*' }); + expect(q).toBe( + "source=logs-* | where `@timestamp` >= '2025-01-01T00:00:00.000Z' and `@timestamp` <= '2025-01-01T01:00:00.000Z' | where level=\"error\"" + ); + }); + + it('leaves the user source clause alone when already present and ignores the spec.index hint', () => { + const q = buildBoundedPPL('source=other-* | head 10', start, end, { index: 'logs-*' }); + expect(q).toBe( + "source=other-* | where `@timestamp` >= '2025-01-01T00:00:00.000Z' and `@timestamp` <= '2025-01-01T01:00:00.000Z' | head 10" + ); + }); + + it('uses a custom timestamp field name in the injected where clause', () => { + const q = buildBoundedPPL('source=logs-* | head 10', start, end, { timestampField: 'time' }); + expect(q).toBe( + "source=logs-* | where `time` >= '2025-01-01T00:00:00.000Z' and `time` <= '2025-01-01T01:00:00.000Z' | head 10" + ); + }); + + it('runs the bound BEFORE a stats command (otherwise @timestamp is gone)', () => { + const q = buildBoundedPPL('source=logs-* | stats count() by service', start, end); + expect(q).toBe( + "source=logs-* | where `@timestamp` >= '2025-01-01T00:00:00.000Z' and `@timestamp` <= '2025-01-01T01:00:00.000Z' | stats count() by service" + ); + }); + + it('runs the bound BEFORE a fields command that excludes @timestamp', () => { + const q = buildBoundedPPL('source=logs-* | fields service, body', start, end); + expect(q).toBe( + "source=logs-* | where `@timestamp` >= '2025-01-01T00:00:00.000Z' and `@timestamp` <= '2025-01-01T01:00:00.000Z' | fields service, body" + ); + }); + + it('runs the bound BEFORE a top command (which collapses to top-N rows without @timestamp)', () => { + const q = buildBoundedPPL('source=logs-* | top 3 service', start, end); + expect(q).toBe( + "source=logs-* | where `@timestamp` >= '2025-01-01T00:00:00.000Z' and `@timestamp` <= '2025-01-01T01:00:00.000Z' | top 3 service" + ); + }); + + it('appends a single where pipe when the user query has no other pipes', () => { + const q = buildBoundedPPL('source=logs-*', start, end); + expect(q).toBe( + "source=logs-* | where `@timestamp` >= '2025-01-01T00:00:00.000Z' and `@timestamp` <= '2025-01-01T01:00:00.000Z'" + ); + }); + + it('preserves a comma-separated multi-source clause', () => { + const q = buildBoundedPPL('source=logs-2026.04,logs-2026.05 | head 10', start, end); + expect(q.startsWith('source=logs-2026.04,logs-2026.05 | where `@timestamp`')).toBe(true); + expect(q).toContain('| head 10'); + }); + + it("preserves the optional 'search' keyword in front of source=", () => { + const q = buildBoundedPPL('search source=logs-* | head 10', start, end); + expect(q.startsWith('search source=logs-* | where `@timestamp`')).toBe(true); + expect(q).toContain('| head 10'); + }); + + it('preserves whitespace around the source equals sign', () => { + const q = buildBoundedPPL('source = logs-* | head 10', start, end); + expect(q.startsWith('source = logs-* | where `@timestamp`')).toBe(true); + expect(q).toContain('| head 10'); + }); + + it('keeps the user time filter intact when one is already present (PPL ANDs them, both run before stats)', () => { + const q = buildBoundedPPL( + "source=logs-* | where `@timestamp` >= '2024-12-31T00:00:00Z' | stats count() by service", + start, + end + ); + expect(q).toBe( + "source=logs-* | where `@timestamp` >= '2025-01-01T00:00:00.000Z' and `@timestamp` <= '2025-01-01T01:00:00.000Z' | where `@timestamp` >= '2024-12-31T00:00:00Z' | stats count() by service" + ); + }); + + it('skips the time-range filter when disableTimeFilter is true', () => { + const q = buildBoundedPPL('source=logs-* | where level="error"', start, end, { disableTimeFilter: true }); + expect(q).toBe('source=logs-* | where level="error"'); + }); + + it('still applies the source= prefix when disableTimeFilter is true', () => { + const q = buildBoundedPPL('where level="error"', start, end, { index: 'logs-*', disableTimeFilter: true }); + expect(q).toBe('source=logs-* | where level="error"'); + }); +}); + +describe('convertPPLToLogs', () => { + it('maps @timestamp and message fields and collects remaining fields as labels', () => { + const logs = convertPPLToLogs({ + schema: [ + { name: '@timestamp', type: 'timestamp' }, + { name: 'message', type: 'text' }, + { name: 'service', type: 'keyword' }, + ], + datarows: [['2025-01-01T00:00:00.000Z', 'hello', 'api']], + }); + expect(logs.entries).toHaveLength(1); + expect(logs.entries[0]?.line).toBe('hello'); + expect(logs.entries[0]?.labels).toEqual({ service: 'api' }); + }); + + it('parses numeric epoch-millis timestamps into seconds', () => { + const logs = convertPPLToLogs({ + schema: [ + { name: '@timestamp', type: 'long' }, + { name: 'message', type: 'text' }, + ], + datarows: [[1735689600000, 'hello']], + }); + expect(logs.entries[0]?.timestamp).toBe(1735689600); + }); + + it('falls back to a JSON dump of the row when no message field is present', () => { + const logs = convertPPLToLogs({ + schema: [ + { name: '@timestamp', type: 'timestamp' }, + { name: 'event', type: 'text' }, + ], + datarows: [['2025-01-01T00:00:00.000Z', 'started']], + }); + expect(logs.entries[0]?.line).toContain('"event":"started"'); + }); + + it('handles empty datarows', () => { + const logs = convertPPLToLogs({ schema: [], datarows: [] }); + expect(logs).toEqual({ entries: [], totalCount: 0 }); + }); + + it('honors explicit timestampField and messageField overrides', () => { + const logs = convertPPLToLogs( + { + schema: [ + { name: 'time', type: 'timestamp' }, + { name: 'body', type: 'text' }, + { name: 'message', type: 'text' }, + ], + datarows: [['2025-01-01T00:00:00.000Z', 'OTel body wins', 'noise']], + }, + { timestampField: 'time', messageField: 'body' } + ); + expect(logs.entries[0]?.line).toBe('OTel body wins'); + expect(logs.entries[0]?.labels).toEqual({ message: 'noise' }); + expect(logs.entries[0]?.timestamp).toBe(new Date('2025-01-01T00:00:00.000Z').getTime() / 1000); + }); + + it('falls back to defaults when the override field is not in the schema', () => { + const logs = convertPPLToLogs( + { + schema: [ + { name: '@timestamp', type: 'timestamp' }, + { name: 'message', type: 'text' }, + ], + datarows: [['2025-01-01T00:00:00.000Z', 'hello']], + }, + { timestampField: 'nope', messageField: 'also-missing' } + ); + expect(logs.entries[0]?.line).toBe('hello'); + }); + + it('puts trace_id columns into labels for trace pivot', () => { + const logs = convertPPLToLogs({ + schema: [ + { name: '@timestamp', type: 'timestamp' }, + { name: 'message', type: 'text' }, + { name: 'traceId', type: 'keyword' }, + ], + datarows: [['2025-01-01T00:00:00.000Z', 'hello', 'abc123']], + }); + expect(logs.entries[0]?.labels.traceId).toBe('abc123'); + }); +}); diff --git a/opensearch/src/queries/opensearch-log-query/get-opensearch-log-data.ts b/opensearch/src/queries/opensearch-log-query/get-opensearch-log-data.ts new file mode 100644 index 000000000..bc657b9e6 --- /dev/null +++ b/opensearch/src/queries/opensearch-log-query/get-opensearch-log-data.ts @@ -0,0 +1,170 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { replaceVariables } from '@perses-dev/plugin-system'; +import { LogEntry, LogData } from '@perses-dev/spec'; +import { OpenSearchClient } from '../../model/opensearch-client'; +import { OpenSearchPPLResponse } from '../../model/opensearch-client-types'; +import { DEFAULT_DATASOURCE, DEFAULT_MESSAGE_FIELDS, DEFAULT_TIMESTAMP_FIELDS } from '../constants'; +import { OpenSearchLogQuerySpec } from './opensearch-log-query-types'; +import { LogQueryPlugin, LogQueryContext } from './log-query-plugin-interface'; + +/** + * Bound the query to the panel time range using a PPL `where` clause on @timestamp. + * The bound is injected immediately after the source clause so it runs before any + * pipe that drops the timestamp column from the schema (stats, fields, top, etc.). + * If the user already filters on @timestamp themselves, PPL ANDs the two clauses. + */ +interface BoundedPPLOptions { + index?: string; + timestampField?: string; + disableTimeFilter?: boolean; +} + +export function buildBoundedPPL( + userQuery: string, + start: Date, + end: Date, + { index, timestampField = '@timestamp', disableTimeFilter = false }: BoundedPPLOptions = {} +): string { + let trimmed = userQuery.trim(); + + if (index && !/^(?:search\s+)?source\s*=/i.test(trimmed)) { + trimmed = `source=${index} | ${trimmed}`; + } + + // Skip the auto-injected time-range clause when the caller manages their own time + // bounds, or when the target index has no usable timestamp field. + if (disableTimeFilter) { + return trimmed; + } + + const startIso = start.toISOString(); + const endIso = end.toISOString(); + const bound = `where \`${timestampField}\` >= '${startIso}' and \`${timestampField}\` <= '${endIso}'`; + + const firstPipe = trimmed.indexOf('|'); + if (firstPipe === -1) { + return `${trimmed} | ${bound}`; + } + + const sourceClause = trimmed.slice(0, firstPipe).trimEnd(); + const rest = trimmed.slice(firstPipe + 1).trimStart(); + return `${sourceClause} | ${bound} | ${rest}`; +} + +function pickIndex(cols: Array<{ name: string }>, candidates: string[]): number { + for (const candidate of candidates) { + const idx = cols.findIndex((c) => c.name === candidate); + if (idx !== -1) return idx; + } + return -1; +} + +interface ConvertOptions { + timestampField?: string; + messageField?: string; +} + +export function convertPPLToLogs(response: OpenSearchPPLResponse, options: ConvertOptions = {}): LogData { + const { schema = [], datarows = [] } = response; + + const tsCandidates = options.timestampField + ? [options.timestampField, ...DEFAULT_TIMESTAMP_FIELDS] + : DEFAULT_TIMESTAMP_FIELDS; + const msgCandidates = options.messageField + ? [options.messageField, ...DEFAULT_MESSAGE_FIELDS] + : DEFAULT_MESSAGE_FIELDS; + + const tsIdx = pickIndex(schema, tsCandidates); + const msgIdx = pickIndex(schema, msgCandidates); + + const entries: LogEntry[] = datarows.map((row) => { + const rawTs = tsIdx !== -1 ? row[tsIdx] : null; + const rawMsg = msgIdx !== -1 ? row[msgIdx] : null; + + const timestamp = parseTimestamp(rawTs); + const line = rawMsg !== null && rawMsg !== undefined ? String(rawMsg) : JSON.stringify(rowToObject(schema, row)); + + const labels: Record = {}; + schema.forEach((col, i) => { + if (i === tsIdx || i === msgIdx) return; + const v = row[i]; + if (v !== null && v !== undefined) labels[col.name] = String(v); + }); + + return { timestamp, line, labels }; + }); + + return { entries, totalCount: entries.length }; +} + +function parseTimestamp(v: unknown): number { + if (v === null || v === undefined) return 0; + if (typeof v === 'number') { + // Heuristic: anything past year ~5138 in seconds-since-epoch (1e11s) must + // really be milliseconds-since-epoch. OpenSearch usually returns ms here. + return v > 1e11 ? v / 1000 : v; + } + const parsed = Date.parse(String(v)); + return Number.isNaN(parsed) ? 0 : parsed / 1000; +} + +function rowToObject( + schema: OpenSearchPPLResponse['schema'], + row: Array +): Record { + const out: Record = {}; + schema.forEach((col, i) => { + out[col.name] = row[i]; + }); + return out; +} + +export const getOpenSearchLogData: LogQueryPlugin['getLogData'] = async ( + spec: OpenSearchLogQuerySpec, + context: LogQueryContext +) => { + if (!spec.query) { + return { + logs: { entries: [], totalCount: 0 }, + timeRange: { start: context.timeRange.start, end: context.timeRange.end }, + }; + } + + const query = replaceVariables(spec.query, context.variableState); + const resolvedIndex = spec.index ? replaceVariables(spec.index, context.variableState) : undefined; + const client = (await context.datasourceStore.getDatasourceClient( + spec.datasource ?? DEFAULT_DATASOURCE + )) as OpenSearchClient; + + const { start, end } = context.timeRange; + const boundedQuery = buildBoundedPPL(query, start, end, { + index: resolvedIndex, + timestampField: spec.timestampField, + disableTimeFilter: spec.disableTimeFilter, + }); + + const response = await client.ppl({ query: boundedQuery }); + + return { + logs: convertPPLToLogs(response, { + timestampField: spec.timestampField, + messageField: spec.messageField, + }), + timeRange: { start, end }, + metadata: { + executedQueryString: boundedQuery, + }, + }; +}; diff --git a/opensearch/src/queries/opensearch-log-query/index.ts b/opensearch/src/queries/opensearch-log-query/index.ts new file mode 100644 index 000000000..58706dec6 --- /dev/null +++ b/opensearch/src/queries/opensearch-log-query/index.ts @@ -0,0 +1,17 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export * from './get-opensearch-log-data'; +export * from './OpenSearchLogQuery'; +export * from './OpenSearchLogQueryEditor'; +export * from './opensearch-log-query-types'; diff --git a/opensearch/src/queries/opensearch-log-query/log-query-plugin-interface.ts b/opensearch/src/queries/opensearch-log-query/log-query-plugin-interface.ts new file mode 100644 index 000000000..fd6717676 --- /dev/null +++ b/opensearch/src/queries/opensearch-log-query/log-query-plugin-interface.ts @@ -0,0 +1,38 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { LogData, AbsoluteTimeRange, UnknownSpec } from '@perses-dev/spec'; +import { DatasourceStore, Plugin, VariableStateMap } from '@perses-dev/plugin-system'; + +export interface LogQueryResult { + logs: LogData; + timeRange: AbsoluteTimeRange; + metadata?: { + executedQueryString: string; + }; +} + +export interface LogQueryContext { + timeRange: AbsoluteTimeRange; + variableState: VariableStateMap; + datasourceStore: DatasourceStore; +} + +type LogQueryPluginDependencies = { + variables?: string[]; +}; + +export interface LogQueryPlugin extends Plugin { + getLogData: (spec: Spec, ctx: LogQueryContext) => Promise; + dependsOn?: (spec: Spec, ctx: LogQueryContext) => LogQueryPluginDependencies; +} diff --git a/opensearch/src/queries/opensearch-log-query/opensearch-log-query-types.ts b/opensearch/src/queries/opensearch-log-query/opensearch-log-query-types.ts new file mode 100644 index 000000000..8c1138667 --- /dev/null +++ b/opensearch/src/queries/opensearch-log-query/opensearch-log-query-types.ts @@ -0,0 +1,27 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { DatasourceSelector } from '@perses-dev/spec'; +import { OpenSearchPPLResponse } from '../../model/opensearch-client-types'; + +export interface OpenSearchLogQuerySpec { + query: string; + datasource?: DatasourceSelector; + index?: string; + timestampField?: string; + messageField?: string; + /** When true, the panel time range is NOT injected as a `where` clause on the timestamp field. */ + disableTimeFilter?: boolean; +} + +export type OpenSearchLogQueryResponse = OpenSearchPPLResponse; diff --git a/opensearch/src/queries/query-editor-model.ts b/opensearch/src/queries/query-editor-model.ts new file mode 100644 index 000000000..f0b5e99b4 --- /dev/null +++ b/opensearch/src/queries/query-editor-model.ts @@ -0,0 +1,56 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { useState } from 'react'; +import { produce } from 'immer'; +import { OptionsEditorProps } from '@perses-dev/plugin-system'; + +type OpenSearchQuerySpec = { + query: string; +}; + +/** + * Keep a local copy of the query input so we don't re-run the panel preview on every keystroke. + * Changes are propagated to the spec on blur. + */ +export function useQueryState( + props: OptionsEditorProps +): { + query: string; + handleQueryChange: (e: string) => void; + handleQueryBlur: () => void; +} { + const { onChange, value } = props; + + const [query, setQuery] = useState(value.query); + const [lastSyncedQuery, setLastSyncedQuery] = useState(value.query); + if (value.query !== lastSyncedQuery) { + setQuery(value.query); + setLastSyncedQuery(value.query); + } + + const handleQueryChange = (e: string): void => { + setQuery(e); + }; + + const handleQueryBlur = (): void => { + setLastSyncedQuery(query); + onChange( + produce(value, (draft) => { + draft.query = query; + }) + ); + }; + + return { query, handleQueryChange, handleQueryBlur }; +} diff --git a/opensearch/src/setup-tests.ts b/opensearch/src/setup-tests.ts new file mode 100644 index 000000000..c4b091083 --- /dev/null +++ b/opensearch/src/setup-tests.ts @@ -0,0 +1,17 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import '@testing-library/jest-dom'; + +// Always mock e-charts during tests since we don't have a proper canvas in jsdom +jest.mock('echarts/core'); diff --git a/opensearch/tsconfig.build.json b/opensearch/tsconfig.build.json new file mode 100644 index 000000000..fc0aafe27 --- /dev/null +++ b/opensearch/tsconfig.build.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "exclude": ["**/*.stories.*", "**/*.test.*", "**/*.map"], + "compilerOptions": { + "emitDeclarationOnly": true, + "declaration": true, + "preserveWatchOutput": true + } +} diff --git a/opensearch/tsconfig.json b/opensearch/tsconfig.json new file mode 100644 index 000000000..98221344c --- /dev/null +++ b/opensearch/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist/lib", + "rootDir": "./src" + }, + "include": ["src"] +} diff --git a/package-lock.json b/package-lock.json index 6d3d10272..76bddd8a5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "logstable", "loki", "markdown", + "opensearch", "piechart", "prometheus", "pyroscope", @@ -4193,6 +4194,10 @@ "resolved": "markdown", "link": true }, + "node_modules/@perses-dev/opensearch-plugin": { + "resolved": "opensearch", + "link": true + }, "node_modules/@perses-dev/pie-chart-plugin": { "resolved": "piechart", "link": true @@ -17366,6 +17371,30 @@ } } }, + "opensearch": { + "name": "@perses-dev/opensearch-plugin", + "version": "0.1.0", + "peerDependencies": { + "@emotion/react": "^11.7.1", + "@emotion/styled": "^11.6.0", + "@hookform/resolvers": "^3.2.0", + "@perses-dev/components": "^0.54.0-beta.3", + "@perses-dev/dashboards": "^0.54.0-beta.3", + "@perses-dev/explore": "^0.54.0-beta.3", + "@perses-dev/plugin-system": "^0.54.0-beta.3", + "@perses-dev/spec": "^0.2.0-beta.2", + "@tanstack/react-query": "^4.39.1", + "date-fns": "^4.1.0", + "date-fns-tz": "^3.2.0", + "echarts": "5.5.0", + "immer": "^10.1.1", + "lodash": "^4.17.21", + "react": "^17.0.2 || ^18.0.0", + "react-dom": "^17.0.2 || ^18.0.0", + "react-hook-form": "^7.52.2", + "use-resize-observer": "^9.0.0" + } + }, "piechart": { "name": "@perses-dev/pie-chart-plugin", "version": "0.14.0-beta.0", diff --git a/package.json b/package.json index 1c3d4f0ce..83f995fb8 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "logstable", "loki", "markdown", + "opensearch", "piechart", "prometheus", "pyroscope",