From 1006df0a536af0150b01a8587f8cda37639d6e90 Mon Sep 17 00:00:00 2001 From: May Lee Date: Thu, 4 Dec 2025 14:30:31 -0500 Subject: [PATCH 1/9] add metrics search syntax --- config/_default/menus/main.en.yaml | 4 +- .../search_syntax/_index.md | 4 + .../logs.md} | 9 +- .../search_syntax/metrics.md | 142 ++++++++++++++++++ 4 files changed, 155 insertions(+), 4 deletions(-) create mode 100644 content/en/observability_pipelines/search_syntax/_index.md rename content/en/observability_pipelines/{search_syntax.md => search_syntax/logs.md} (97%) create mode 100644 content/en/observability_pipelines/search_syntax/metrics.md diff --git a/config/_default/menus/main.en.yaml b/config/_default/menus/main.en.yaml index a769718f350..875e5c72821 100644 --- a/config/_default/menus/main.en.yaml +++ b/config/_default/menus/main.en.yaml @@ -6064,9 +6064,9 @@ menu: identifier: observability_pipelines_packs_zscaler_zia_web_logs weight: 520 - name: Search Syntax - url: observability_pipelines/search_syntax/ + url: observability_pipelines/search_syntax/logs/ parent: observability_pipelines - identifier: observability_pipelines_search_syntax + identifier: observability_pipelines_logs_search_syntax weight: 6 - name: Scaling and Performance url: observability_pipelines/scaling_and_performance/ diff --git a/content/en/observability_pipelines/search_syntax/_index.md b/content/en/observability_pipelines/search_syntax/_index.md new file mode 100644 index 00000000000..0816c75d816 --- /dev/null +++ b/content/en/observability_pipelines/search_syntax/_index.md @@ -0,0 +1,4 @@ +--- +title: Search Syntax +type: multi-code-lang +--- \ No newline at end of file diff --git a/content/en/observability_pipelines/search_syntax.md b/content/en/observability_pipelines/search_syntax/logs.md similarity index 97% rename from content/en/observability_pipelines/search_syntax.md rename to content/en/observability_pipelines/search_syntax/logs.md index 8c98c5bff68..31412fae3ff 100644 --- a/content/en/observability_pipelines/search_syntax.md +++ b/content/en/observability_pipelines/search_syntax/logs.md @@ -1,7 +1,12 @@ --- -title: Search Syntax -description: Learn the new filter query search syntax for your Observability Pipelines processors. +title: Logs Search Syntax +description: Learn how to use logs search syntax for your Observability Pipelines processors filter queries. disable_toc: false +aliases: + - /observability_pipelines/search_syntax/ +code_lang: logs +type: multi-code-lang +weight: 1 --- ## Overview diff --git a/content/en/observability_pipelines/search_syntax/metrics.md b/content/en/observability_pipelines/search_syntax/metrics.md new file mode 100644 index 00000000000..32dc6b04ad9 --- /dev/null +++ b/content/en/observability_pipelines/search_syntax/metrics.md @@ -0,0 +1,142 @@ +--- +title: Metrics Search Syntax +description: Learn about how to use metrics search syntax for your Observability Pipelines processors filter queries. +disable_toc: false +code_lang: metrics +type: multi-code-lang +weight: 2 +--- + +## Overview + +When you add a processor to a pipeline, you can filter metrics to only process a defined subset. This document goes over the following information: + +- [Free text search](#free-text-search) +- [Attribute search](#attribute-search) +- [Tags search](#tags-search) +- [Boolean operators](#boolean-operators) +- [Wildcards](#wildcards) + +## Free text search + +Free text search for metrics only searches the `name` field and is case sensitive. + +An example of a free text search: `system.cpu.user`, which matches metrics with `name:system.cpu.user`. + +## Attribute search + +You can search the attribute keys: + +- `kind`: Whether the metrics is `absolute` or `incremental`. + - `absolute` metrics set the reference value for future updates. + - `incremental` metrics update previous values. +- `value`: The metric type. + - `counter` + - `gauge` + - `distribution` + - `histogram` + +Here are some attribute search syntax examples: + +`kind:absolute` +: Matches metrics with `kind:absolute`. + +`value:counter` +: Matches metrics with `counter` metrics. + +**Note**: Attribute searches are case sensitive. + +## Tags search {#tags-search} + +You can search a metric's `tags`. Special characters and spaces in a tag must be escaped in the search syntax. + +### Escape special characters and spaces + +The following characters are considered special and must be escaped with a backslash (`\`): + +`-` `!` `&&` `||` `>` `>=` `<` `<=` `(` `)` `{` `}` `[` `]` `"` `*` `?` `:` `#`, and spaces. + +**Notes**: + +\- `/` is not considered a special character and doesn't need to be escaped. +\- You can search for special characters in tag. See [Search a tag that contains special characters](#search-a-tag-that-contains-special-characters). +\- If you want to match metrics tags that contain the special character `!` , use the attribute search syntax: `tags:*/:!*.` + +### Search a tag that contains special characters + +Searching for a tag that contains special characters requires escaping or double quotes. For example, to search for a tag `my_app` with the value `hello:world`, use one of the following search syntaxes: + +- `tags\:my_app:hello\:world` +- `tags:\my_app:"hello:world"` + +### Match a single special character + +To match a single special character or space, use the `?` wildcard. For example, the search syntax: `"tags\:message\:hello?world"` matches metrics with either of these tags: + +- `message:"hello world"` +- `message:"hello-world"`. + +### Examples + +To learn how to escape special characters in a tag search, let's look at a metric example: + +``` +{ +"name":"datadog.agent.retry_queue_duration.bytes_per_sec", + "tags":{ + "agent":"core", + "domain":"https://7-72-3-app.agent.datadoghq.com", + "host":"COMP-YGVQDJG75L", + "source_type_name":"System", + "env:prod" + }, + "timestamp":"2025-11-28T13:03:09Z", + "kind":"absolute", + "gauge":{"value":454.1372767857143} +} +``` + +The following are search syntax examples that escape special character in the metric example: + +`tags:env\*` +: Matches metrics with `tags` containing the `env` key. + +`tags:(env\:prod OR env\:test)` +: Matches metrics with `env:prod` or `env:test` in `tags`. +: This query can also be written as `tags:("env:prod" OR "env:test")`. + +`tags:env\:prod AND -tags:version\:beta` +: Matches metrics that have `env:prod` and does not have `version:beta` in `tags`. +: This query can also be written as `tags:"env:prod" AND -tags:"version:beta"`. + +## Boolean operators + +You can use the following case sensitive Boolean operators to combine multiple terms in a search query. + +The follow are example queries that use Boolean operators: + +`NOT system.cpu.user` +: Matches metrics that do not have `name:system.cpu.user`. +: This query can also be written as `-system.cpu.user`. + +`system.cpu.user OR system.cpu.user.total` +: Matches metrics that have either `name:system.cpu.user` or `name:system.cpu.user.total`. + +`value:counter AND (tags:env/:staging OR tags:env/:dev)` +: Matches metrics that have both `counter:{value:}` and the either the tag `env:prod` or `env:dev` + +## Wildcards + +​​You can use `*` for wildcard searches. The following are wildcard search examples: + +`*system.cpu.user*` +: Matches metrics with a `name` field value that contains `system.cpu.user`. + +`system.cpu.user*` +: Matches metrics with a `name` field value that starts with `system.cpu.user`. + +`*system.cpu.user` +: Matches metrics with a `name` field value that ends with `system.cpu.user`. + +`tags:*\:bin` +: Matches metrics that have a tag with the value `bin`, regardless of what the tag key is. From fdd1bb48b150b8661a3593b474d97194b02b5321 Mon Sep 17 00:00:00 2001 From: May Lee Date: Thu, 4 Dec 2025 14:36:26 -0500 Subject: [PATCH 2/9] small edit --- content/en/observability_pipelines/search_syntax/metrics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/observability_pipelines/search_syntax/metrics.md b/content/en/observability_pipelines/search_syntax/metrics.md index 32dc6b04ad9..e25ebb414c0 100644 --- a/content/en/observability_pipelines/search_syntax/metrics.md +++ b/content/en/observability_pipelines/search_syntax/metrics.md @@ -9,7 +9,7 @@ weight: 2 ## Overview -When you add a processor to a pipeline, you can filter metrics to only process a defined subset. This document goes over the following information: +When you add a processor to a pipeline, you can filter metrics ({{< tooltip glossary="preview" >}}) to only process a defined subset. This document goes over the following information: - [Free text search](#free-text-search) - [Attribute search](#attribute-search) From f0ec7b1b4d652a36e16ab13f9c1c909a801e3882 Mon Sep 17 00:00:00 2001 From: May Lee Date: Thu, 4 Dec 2025 15:37:33 -0500 Subject: [PATCH 3/9] edits --- .../search_syntax/metrics.md | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/content/en/observability_pipelines/search_syntax/metrics.md b/content/en/observability_pipelines/search_syntax/metrics.md index e25ebb414c0..08c490985fa 100644 --- a/content/en/observability_pipelines/search_syntax/metrics.md +++ b/content/en/observability_pipelines/search_syntax/metrics.md @@ -28,8 +28,6 @@ An example of a free text search: `system.cpu.user`, which matches metrics with You can search the attribute keys: - `kind`: Whether the metrics is `absolute` or `incremental`. - - `absolute` metrics set the reference value for future updates. - - `incremental` metrics update previous values. - `value`: The metric type. - `counter` - `gauge` @@ -42,11 +40,11 @@ Here are some attribute search syntax examples: : Matches metrics with `kind:absolute`. `value:counter` -: Matches metrics with `counter` metrics. +: Matches `counter` metrics. **Note**: Attribute searches are case sensitive. -## Tags search {#tags-search} +## Tags search You can search a metric's `tags`. Special characters and spaces in a tag must be escaped in the search syntax. @@ -58,20 +56,20 @@ The following characters are considered special and must be escaped with a backs **Notes**: -\- `/` is not considered a special character and doesn't need to be escaped. -\- You can search for special characters in tag. See [Search a tag that contains special characters](#search-a-tag-that-contains-special-characters). -\- If you want to match metrics tags that contain the special character `!` , use the attribute search syntax: `tags:*/:!*.` +- `/` is not considered a special character and doesn't need to be escaped. +- You can search for special characters in tag. See [Search a tag that contains special characters](#search-a-tag-that-contains-special-characters). +- If you want to match metrics tags that contain the special character `!` , use the attribute search syntax: `tags:*\:!*.` ### Search a tag that contains special characters Searching for a tag that contains special characters requires escaping or double quotes. For example, to search for a tag `my_app` with the value `hello:world`, use one of the following search syntaxes: -- `tags\:my_app:hello\:world` -- `tags:\my_app:"hello:world"` +- `tags:my_app:hello\:world` +- `tags:my_app:"hello:world"` ### Match a single special character -To match a single special character or space, use the `?` wildcard. For example, the search syntax: `"tags\:message\:hello?world"` matches metrics with either of these tags: +To match a single special character or space, use the `?` wildcard. For example, the search syntax: `"tags:message\:hello?world"` matches metrics with either of these tags: - `message:"hello world"` - `message:"hello-world"`. @@ -96,9 +94,9 @@ To learn how to escape special characters in a tag search, let's look at a metri } ``` -The following are search syntax examples that escape special character in the metric example: +The following are search syntax examples that escape special characters in the metric example: -`tags:env\*` +`tags:env*` : Matches metrics with `tags` containing the `env` key. `tags:(env\:prod OR env\:test)` @@ -122,7 +120,7 @@ The follow are example queries that use Boolean operators: `system.cpu.user OR system.cpu.user.total` : Matches metrics that have either `name:system.cpu.user` or `name:system.cpu.user.total`. -`value:counter AND (tags:env/:staging OR tags:env/:dev)` +`value:counter AND (tags:env\:staging OR tags:env\:dev)` : Matches metrics that have both `counter:{value:}` and the either the tag `env:prod` or `env:dev` ## Wildcards @@ -130,13 +128,13 @@ The follow are example queries that use Boolean operators: ​​You can use `*` for wildcard searches. The following are wildcard search examples: `*system.cpu.user*` -: Matches metrics with a `name` field value that contains `system.cpu.user`. +: Matches metrics with a `name` value that contains `system.cpu.user`. `system.cpu.user*` -: Matches metrics with a `name` field value that starts with `system.cpu.user`. +: Matches metrics with a `name` value that starts with `system.cpu.user`. `*system.cpu.user` -: Matches metrics with a `name` field value that ends with `system.cpu.user`. +: Matches metrics with a `name` value that ends with `system.cpu.user`. `tags:*\:bin` : Matches metrics that have a tag with the value `bin`, regardless of what the tag key is. From 8e8d07a3e54fc664113351257176d9363b0c1210 Mon Sep 17 00:00:00 2001 From: May Lee Date: Thu, 4 Dec 2025 17:33:27 -0500 Subject: [PATCH 4/9] Apply suggestions from code review Co-authored-by: Michael Cretzman <58786311+michaelcretzman@users.noreply.github.com> --- content/en/observability_pipelines/search_syntax/metrics.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/en/observability_pipelines/search_syntax/metrics.md b/content/en/observability_pipelines/search_syntax/metrics.md index 08c490985fa..14d659c08ff 100644 --- a/content/en/observability_pipelines/search_syntax/metrics.md +++ b/content/en/observability_pipelines/search_syntax/metrics.md @@ -57,10 +57,10 @@ The following characters are considered special and must be escaped with a backs **Notes**: - `/` is not considered a special character and doesn't need to be escaped. -- You can search for special characters in tag. See [Search a tag that contains special characters](#search-a-tag-that-contains-special-characters). +- You can search for special characters in a tag. See [Search for a tag that contains special characters](#search-for-a-tag-that-contains-special-characters). - If you want to match metrics tags that contain the special character `!` , use the attribute search syntax: `tags:*\:!*.` -### Search a tag that contains special characters +### Search for a tag that contains special characters Searching for a tag that contains special characters requires escaping or double quotes. For example, to search for a tag `my_app` with the value `hello:world`, use one of the following search syntaxes: From 3f394e921662976b3d40ad40d77f18934ac8fc23 Mon Sep 17 00:00:00 2001 From: May Lee Date: Mon, 8 Dec 2025 11:00:04 -0500 Subject: [PATCH 5/9] small edit --- content/en/observability_pipelines/search_syntax/metrics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/observability_pipelines/search_syntax/metrics.md b/content/en/observability_pipelines/search_syntax/metrics.md index 14d659c08ff..20e90fe01d9 100644 --- a/content/en/observability_pipelines/search_syntax/metrics.md +++ b/content/en/observability_pipelines/search_syntax/metrics.md @@ -9,7 +9,7 @@ weight: 2 ## Overview -When you add a processor to a pipeline, you can filter metrics ({{< tooltip glossary="preview" >}}) to only process a defined subset. This document goes over the following information: +When you add a processor to a pipeline, you can filter metrics ({{< tooltip text="Preview" glossary="preview" case="title" >}}) to only process a defined subset. This document goes over the following information: - [Free text search](#free-text-search) - [Attribute search](#attribute-search) From 1b1c299d7cd6b9dae412be7bd296abbad3cb16d2 Mon Sep 17 00:00:00 2001 From: May Lee Date: Mon, 8 Dec 2025 11:15:32 -0500 Subject: [PATCH 6/9] small edit --- content/en/observability_pipelines/search_syntax/metrics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/observability_pipelines/search_syntax/metrics.md b/content/en/observability_pipelines/search_syntax/metrics.md index 20e90fe01d9..2c27e96cf90 100644 --- a/content/en/observability_pipelines/search_syntax/metrics.md +++ b/content/en/observability_pipelines/search_syntax/metrics.md @@ -9,7 +9,7 @@ weight: 2 ## Overview -When you add a processor to a pipeline, you can filter metrics ({{< tooltip text="Preview" glossary="preview" case="title" >}}) to only process a defined subset. This document goes over the following information: +When you add a processor to a pipeline, you can filter metrics ({{< tooltip glossary="preview" case="title" >}}) to only process a defined subset. This document goes over the following information: - [Free text search](#free-text-search) - [Attribute search](#attribute-search) From ab526e1c447d27fbb6b52eb0231d03ea4b561fa6 Mon Sep 17 00:00:00 2001 From: May Lee Date: Fri, 12 Dec 2025 11:18:31 -0500 Subject: [PATCH 7/9] small edits --- content/en/observability_pipelines/search_syntax/metrics.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/content/en/observability_pipelines/search_syntax/metrics.md b/content/en/observability_pipelines/search_syntax/metrics.md index 2c27e96cf90..d37810afcb4 100644 --- a/content/en/observability_pipelines/search_syntax/metrics.md +++ b/content/en/observability_pipelines/search_syntax/metrics.md @@ -64,15 +64,15 @@ The following characters are considered special and must be escaped with a backs Searching for a tag that contains special characters requires escaping or double quotes. For example, to search for a tag `my_app` with the value `hello:world`, use one of the following search syntaxes: -- `tags:my_app:hello\:world` -- `tags:my_app:"hello:world"` +- `tags:my_app\:hello\:world` +- `tags:my_app\:"hello:world"` ### Match a single special character To match a single special character or space, use the `?` wildcard. For example, the search syntax: `"tags:message\:hello?world"` matches metrics with either of these tags: - `message:"hello world"` -- `message:"hello-world"`. +- `message:"hello-world"` ### Examples From f502cec018cca1620f58339fc01b6923ffb0d3ef Mon Sep 17 00:00:00 2001 From: May Lee Date: Mon, 15 Dec 2025 10:07:39 -0500 Subject: [PATCH 8/9] fix indentation --- .../en/observability_pipelines/search_syntax/metrics.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/content/en/observability_pipelines/search_syntax/metrics.md b/content/en/observability_pipelines/search_syntax/metrics.md index d37810afcb4..2c99f35037f 100644 --- a/content/en/observability_pipelines/search_syntax/metrics.md +++ b/content/en/observability_pipelines/search_syntax/metrics.md @@ -81,16 +81,16 @@ To learn how to escape special characters in a tag search, let's look at a metri ``` { "name":"datadog.agent.retry_queue_duration.bytes_per_sec", - "tags":{ +"tags":{ "agent":"core", "domain":"https://7-72-3-app.agent.datadoghq.com", "host":"COMP-YGVQDJG75L", "source_type_name":"System", "env:prod" }, - "timestamp":"2025-11-28T13:03:09Z", - "kind":"absolute", - "gauge":{"value":454.1372767857143} +"timestamp":"2025-11-28T13:03:09Z", +"kind":"absolute", +"gauge":{"value":454.1372767857143} } ``` From 48ff54a84ea30d11d019fad1d622b22e5773b503 Mon Sep 17 00:00:00 2001 From: May Lee Date: Mon, 15 Dec 2025 10:21:01 -0500 Subject: [PATCH 9/9] fix indentation --- .../search_syntax/metrics.md | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/content/en/observability_pipelines/search_syntax/metrics.md b/content/en/observability_pipelines/search_syntax/metrics.md index 2c99f35037f..c83771bbe2a 100644 --- a/content/en/observability_pipelines/search_syntax/metrics.md +++ b/content/en/observability_pipelines/search_syntax/metrics.md @@ -80,17 +80,17 @@ To learn how to escape special characters in a tag search, let's look at a metri ``` { -"name":"datadog.agent.retry_queue_duration.bytes_per_sec", -"tags":{ - "agent":"core", - "domain":"https://7-72-3-app.agent.datadoghq.com", - "host":"COMP-YGVQDJG75L", - "source_type_name":"System", - "env:prod" - }, -"timestamp":"2025-11-28T13:03:09Z", -"kind":"absolute", -"gauge":{"value":454.1372767857143} + "name":"datadog.agent.retry_queue_duration.bytes_per_sec", + "tags":{ + "agent":"core", + "domain":"https://7-72-3-app.agent.datadoghq.com", + "host":"COMP-YGVQDJG75L", + "source_type_name":"System", + "env:prod" + }, + "timestamp":"2025-11-28T13:03:09Z", + "kind":"absolute", + "gauge":{"value":454.1372767857143} } ```