From 57f4c40696b31abb555dc81d4160c42cc2b7e383 Mon Sep 17 00:00:00 2001 From: Lucas Sampaio Date: Tue, 12 May 2026 16:58:27 -0700 Subject: [PATCH 1/3] Add WIP docs for eBPF L7 observability New L7 observability pages for the eBPF-based collector (tigera/calico-private#11749): - configure-bpf.mdx: enable via L7ObservabilityEnabled on FelixConfiguration; comparison with the Istio Waypoint collector - configure-istio-waypoint.mdx: placeholder - datatypes-wip.mdx: redraft of the L7 schema with a Populated by column and new protocol / collector_name / tls_* fields - aggregation.mdx: aggregation key fields with an interactive playground component - sidebar entries for all four pages Co-Authored-By: Claude Opus 4.7 (1M context) --- .../components/L7AggregationSandbox/index.js | 150 ++++++++++++++++++ .../observability/elastic/l7/aggregation.mdx | 41 +++++ .../elastic/l7/configure-bpf.mdx | 78 +++++++++ .../elastic/l7/configure-istio-waypoint.mdx | 23 +++ .../elastic/l7/datatypes-wip.mdx | 57 +++++++ sidebars-calico-enterprise.js | 9 +- 6 files changed, 357 insertions(+), 1 deletion(-) create mode 100644 calico-enterprise/_includes/components/L7AggregationSandbox/index.js create mode 100644 calico-enterprise/observability/elastic/l7/aggregation.mdx create mode 100644 calico-enterprise/observability/elastic/l7/configure-bpf.mdx create mode 100644 calico-enterprise/observability/elastic/l7/configure-istio-waypoint.mdx create mode 100644 calico-enterprise/observability/elastic/l7/datatypes-wip.mdx diff --git a/calico-enterprise/_includes/components/L7AggregationSandbox/index.js b/calico-enterprise/_includes/components/L7AggregationSandbox/index.js new file mode 100644 index 0000000000..18379a124e --- /dev/null +++ b/calico-enterprise/_includes/components/L7AggregationSandbox/index.js @@ -0,0 +1,150 @@ +import React, { useState, useMemo } from 'react'; + +const SAMPLE = [ + { src: 'frontend', dest: 'cart', svc: 'cart', method: 'GET', url: '/cart', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, + { src: 'frontend', dest: 'cart', svc: 'cart', method: 'GET', url: '/cart', code: '200', ua: 'curl/8.0', protocol: 'http', sni: null }, + { src: 'frontend', dest: 'cart', svc: 'cart', method: 'PUT', url: '/cart', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, + { src: 'frontend', dest: 'cart', svc: 'cart', method: 'GET', url: '/products', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, + { src: 'frontend', dest: 'cart', svc: 'cart', method: 'GET', url: '/cart', code: '404', ua: 'cart-client/1.0', protocol: 'http', sni: null }, + { src: 'checkout', dest: 'cart', svc: 'cart', method: 'GET', url: '/cart', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, + { src: 'frontend', dest: 'cart-canary', svc: 'cart', method: 'GET', url: '/cart', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, + { src: 'frontend', dest: 'cart', svc: 'cart-v2', method: 'GET', url: '/cart', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, + { src: 'frontend', dest: 'cart', svc: 'cart', method: null, url: null, code: null, ua: null, protocol: 'tls', sni: 'cart.default.svc' }, + { src: 'frontend', dest: 'cart', svc: 'cart', method: null, url: null, code: null, ua: null, protocol: 'tls', sni: 'payment.default.svc' }, +]; + +const AGGREGATORS = [ + { id: 'source', label: 'Source info', fields: ['src'], field: 'L7LogsFileAggregationSourceInfo' }, + { id: 'dest', label: 'Destination info', fields: ['dest'], field: 'L7LogsFileAggregationDestinationInfo' }, + { id: 'service', label: 'Service info', fields: ['svc'], field: 'L7LogsFileAggregationServiceInfo' }, + { id: 'method', label: 'HTTP method', fields: ['method'], field: 'L7LogsFileAggregationHTTPMethod' }, + { id: 'url', label: 'URL', fields: ['url'], field: 'L7LogsFileAggregationTrimURL' }, + { id: 'code', label: 'Response code', fields: ['code'], field: 'L7LogsFileAggregationResponseCode' }, + { id: 'header', label: 'HTTP header info', fields: ['ua'], field: 'L7LogsFileAggregationHTTPHeaderInfo' }, + { id: 'tlssni', label: 'TLS SNI', fields: ['sni'], field: 'L7LogsFileAggregationTLSSNI' }, +]; + +const COLUMNS = [ + { key: 'protocol', label: 'protocol', owner: null }, + { key: 'src', label: 'src_name_aggr', owner: 'source' }, + { key: 'dest', label: 'dest_name_aggr', owner: 'dest' }, + { key: 'svc', label: 'dest_service_name', owner: 'service' }, + { key: 'method', label: 'method', owner: 'method' }, + { key: 'url', label: 'url', owner: 'url' }, + { key: 'code', label: 'response_code', owner: 'code' }, + { key: 'ua', label: 'user_agent', owner: 'header' }, + { key: 'sni', label: 'tls_sni', owner: 'tlssni' }, +]; + +export default function L7AggregationSandbox() { + const [checked, setChecked] = useState( + Object.fromEntries(AGGREGATORS.map((a) => [a.id, true])), + ); + + const aggregated = useMemo(() => { + const buckets = new Map(); + for (const row of SAMPLE) { + const keyParts = [`protocol:${row.protocol}`]; + for (const a of AGGREGATORS) { + if (!checked[a.id]) continue; + for (const f of a.fields) { + keyParts.push(`${f}:${row[f] ?? ''}`); + } + } + const key = keyParts.join('|'); + if (buckets.has(key)) { + buckets.get(key).count += 1; + } else { + buckets.set(key, { ...row, count: 1 }); + } + } + return Array.from(buckets.values()); + }, [checked]); + + const toggle = (id) => + setChecked((c) => ({ ...c, [id]: !c[id] })); + + const cellValue = (row, col) => { + if (col.owner && !checked[col.owner]) return '—'; + return row[col.key] ?? '—'; + }; + + const cellStyle = { + padding: '4px 8px', + fontFamily: 'var(--ifm-font-family-monospace)', + fontSize: '0.85em', + overflowWrap: 'anywhere', + verticalAlign: 'top', + }; + const headerStyle = { + ...cellStyle, + whiteSpace: 'normal', + wordBreak: 'break-word', + }; + + return ( +
+
+ {AGGREGATORS.map((a) => ( + + ))} +
+
+ + + {COLUMNS.map((c) => ( + + ))} + + + + + {COLUMNS.map((c) => ( + + ))} + + + + + {aggregated.map((row, i) => ( + + {COLUMNS.map((c) => ( + + ))} + + + ))} + +
{c.label}count
{cellValue(row, c)}{row.count}
+
+

+ Uncheck a field to drop it from the aggregation key. Rows that differ only in dropped fields merge, and the count column increments accordingly. +

+
+ ); +} diff --git a/calico-enterprise/observability/elastic/l7/aggregation.mdx b/calico-enterprise/observability/elastic/l7/aggregation.mdx new file mode 100644 index 0000000000..431306c050 --- /dev/null +++ b/calico-enterprise/observability/elastic/l7/aggregation.mdx @@ -0,0 +1,41 @@ +--- +description: Control how Calico Enterprise aggregates L7 log entries by selecting which fields participate in the aggregation key. +--- + +import L7AggregationSandbox from '../../../_includes/components/L7AggregationSandbox'; + +# L7 log aggregation + +## Big picture + +$[prodname] aggregates L7 events before writing them to disk. Each row in the resulting log represents a unique combination of metadata fields and carries a `count` of how many requests matched that combination during the aggregation interval. Which fields participate in that combination — the aggregation key — is controlled by a set of fields on `FelixConfiguration`. + +Including a field makes the logs more granular (more rows, fewer requests per row). Excluding a field collapses requests that differ only in that field into a single row and increments its `count` for each merged request. + +## Aggregation fields + +The following `FelixConfiguration` fields control the L7 log aggregation key. For accepted values and defaults, follow the link to the [Felix Configuration reference](../../../reference/component-resources/node/felix/configuration.mdx#l7-logs). + +| Field | Effect when included | Effect when excluded | +| --- | --- | --- | +| [L7LogsFileAggregationSourceInfo](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationSourceInfo) | Source aggregated name, namespace, and type appear on each row. | Requests are merged across sources. | +| [L7LogsFileAggregationDestinationInfo](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationDestinationInfo) | Destination aggregated name, namespace, and type appear on each row. | Requests are merged across destinations. | +| [L7LogsFileAggregationServiceInfo](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationServiceInfo) | Destination service name, namespace, and port appear on each row. | Requests are merged across services. | +| [L7LogsFileAggregationHTTPMethod](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationHTTPMethod) | HTTP method appears on each row. | Requests are merged across methods. | +| [L7LogsFileAggregationTrimURL](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationTrimURL) | URL (or a trimmed prefix) appears on each row. Intermediate values trim the query string, then the path, before excluding the URL entirely. | Requests are merged across URLs. | +| [L7LogsFileAggregationResponseCode](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationResponseCode) | Response code appears on each row. | Requests are merged across response codes. | +| [L7LogsFileAggregationHTTPHeaderInfo](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationHTTPHeaderInfo) | User agent and request type appear on each row. | Requests are merged across user agents and types. | +| [L7LogsFileAggregationTLSSNI](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationTLSSNI) | TLS Server Name Indication appears on each TLS row. | TLS flows to the same destination are merged regardless of SNI. | + +Two additional fields control how the URL is truncated before it enters the aggregation key, rather than whether the URL participates at all: + +| Field | Description | +| --- | --- | +| [L7LogsFileAggregationNumURLPath](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationNumURLPath) | Maximum number of URL path components retained. Default `5`. A negative value disables this truncation. | +| [L7LogsFileAggregationURLCharLimit](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationURLCharLimit) | Maximum URL length in characters; longer URLs are sliced. Default `250`. | + +## Aggregation playground + +The table below shows a small set of sample L7 events — eight HTTP requests and two TLS handshakes. Toggle the checkboxes to include or exclude each field from the aggregation key. The table will collapse and the `count` column will update to show how many original requests fall into each aggregated row. + + diff --git a/calico-enterprise/observability/elastic/l7/configure-bpf.mdx b/calico-enterprise/observability/elastic/l7/configure-bpf.mdx new file mode 100644 index 0000000000..c8ba965228 --- /dev/null +++ b/calico-enterprise/observability/elastic/l7/configure-bpf.mdx @@ -0,0 +1,78 @@ +--- +description: Capture HTTP and TLS L7 logs from kernel TCP probes using eBPF, with no proxy or sidecar injection required. +--- + +# Configure L7 logs with eBPF + +## Big picture + +Use $[prodname] eBPF-based L7 observability to capture HTTP request/response and TLS handshake metadata directly from the kernel TCP layer, without deploying a proxy or injecting sidecars. + +## Value + +L7 logs give platform operators and development teams visibility into how applications talk to each other beyond what L3/4 flow logs show: HTTP method, URL, response code, and TLS SNI / version / cipher suite. They are also key for spotting anomalous behavior such as attempts to access restricted URLs or scans for particular paths. + +The eBPF-based collector delivers this visibility with no data-path proxy and no per-deployment opt-in: enable it once at the cluster level, and HTTP and TLS metadata is captured for every workload on the node. The feature is **dataplane-independent** and works with the eBPF, iptables, and nftables dataplanes alike. + +## Concepts + +### About eBPF L7 logs + +$[prodname] attaches eBPF probes to the kernel's TCP layer and extracts HTTP request/response and TLS handshake metadata from the traffic as it flows through the kernel. The parsed metadata is forwarded to the L7 log pipeline and written to disk on each node. + +Each L7 log entry carries a `protocol` field (`http` or `tls`) and a `collector_name` field (`ebpf-tcp`) so events can be distinguished by their source. TLS entries additionally include `tls_sni`, `tls_version`, and `tls_cipher_suite`. For the full schema, see [L7 log data types](datatypes-wip.mdx). + +### Comparison with the Istio Waypoint collector + +The [Istio Waypoint collector](configure-istio-waypoint.mdx) routes selected traffic through a per-namespace Envoy waypoint proxy and produces L7 logs from the proxy itself. The eBPF collector runs inside Felix on every node and captures HTTP and TLS metadata as traffic passes through the kernel TCP layer. The two collectors can coexist in the same cluster; both feed the same L7 log pipeline and entries are distinguished by the `collector_name` field. + +| Aspect | Istio Waypoint collector | eBPF L7 collector | +| --- | --- | --- | +| Additional infrastructure | Waypoint Envoy pods per namespace/service | None — runs inside Felix | +| L7 coverage | Only traffic routed through a waypoint | All TCP traffic on the node | +| Network path overhead | Two extra proxy hops | None | +| Plaintext access | Decrypted at the waypoint (mTLS terminated) | Captured at the kernel TCP layer, before TLS encryption | +| Protocol coverage | HTTP/1, HTTP/2, gRPC | HTTP/1.0, HTTP/1.1, TLS handshake metadata | +| Opt-in granularity | Per namespace/service | Cluster-wide via FelixConfiguration | +| L7 features beyond observability | Retries, traffic splitting, fault injection | Observability only | + +## Before you begin + +### Limitations + +* Requires Linux kernel 5.17 or later on every node. +* HTTP parsing covers HTTP/1.0 and HTTP/1.1 only, because their requests and responses travel as plaintext on the wire. HTTP/2 and HTTP/3 are not captured as HTTP, and encrypted HTTP traffic surfaces only as TLS metadata. +* HTTP captures only the most recent request per socket; rapidly pipelined requests on a single connection may be undercounted. +* TLS capture is limited to handshake metadata (SNI, version, cipher suite). The collector does not decrypt application traffic. + +:::note + +L7 logs require a minimum of 1 additional GB of log storage per node, per one-day retention period. Adjust your [Log Storage](../../../operations/logstorage/adjust-log-storage-size.mdx) before you start tasks in the next section. + +::: + +## Configure L7 logs + +Enable the eBPF L7 collector by setting the [L7ObservabilityEnabled](../../../reference/component-resources/node/felix/configuration.mdx#L7ObservabilityEnabled) field to `true` on the default `FelixConfiguration`. The setting is independent of `bpfEnabled` — the collector works with any dataplane. + +```bash +kubectl patch felixconfiguration default --type=merge -p '{"spec":{"l7ObservabilityEnabled":true}}' +``` + +To disable the collector, set the field back to `false`. Felix tears down its BPF programs and cleans up pinned maps; no node reboot is required. + +```bash +kubectl patch felixconfiguration default --type=merge -p '{"spec":{"l7ObservabilityEnabled":false}}' +``` + +## View L7 logs + +The eBPF collector writes L7 events as JSON to a log file on each node. By default the file is at `/var/log/calico/l7logs/l7.log`. The directory is controlled by the [L7LogsFileDirectory](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileDirectory) field on `FelixConfiguration`. + +To tail the log on a node: + +```bash +tail -f /var/log/calico/l7logs/l7.log +``` + +For the JSON schema, see [L7 log data types](datatypes-wip.mdx). diff --git a/calico-enterprise/observability/elastic/l7/configure-istio-waypoint.mdx b/calico-enterprise/observability/elastic/l7/configure-istio-waypoint.mdx new file mode 100644 index 0000000000..69a13ef22e --- /dev/null +++ b/calico-enterprise/observability/elastic/l7/configure-istio-waypoint.mdx @@ -0,0 +1,23 @@ +--- +description: Capture L7 logs through an Istio Ambient Mode waypoint proxy. +--- + +# Configure L7 logs with Istio Waypoint + +:::note + +This page is a placeholder. Content for L7 log collection via Istio Ambient Mode waypoint proxies is pending. + +::: + +## Big picture + +## Value + +## Concepts + +## Before you begin + +## Configure L7 logs + +## View L7 logs in the web console diff --git a/calico-enterprise/observability/elastic/l7/datatypes-wip.mdx b/calico-enterprise/observability/elastic/l7/datatypes-wip.mdx new file mode 100644 index 0000000000..e47fca38db --- /dev/null +++ b/calico-enterprise/observability/elastic/l7/datatypes-wip.mdx @@ -0,0 +1,57 @@ +--- +description: Reference of key/value fields that Calico Enterprise sends to Elasticsearch for L7 logs, with the collector that populates each field. +--- + +# L7 log data types (WIP) + +:::note + +This page is a work-in-progress redraft of [L7 log data types](datatypes.mdx). It adds the fields produced by the eBPF L7 collector and a **Populated by** column that identifies which collector fills each field. + +::: + +## Big picture + +$[prodname] supports multiple L7 collectors that write into the same Elasticsearch index (`tigera_secure_ee_l7`): + +- **Envoy** — the legacy sidecar/proxy-based collector. See [Configure L7 logs](configure.mdx). +- **eBPF** — kernel TCP-layer probes capturing HTTP request/response and TLS handshake metadata. See [Configure L7 logs with eBPF](configure-bpf.mdx). +- **Istio Waypoint** — collection through an Istio Ambient Mode waypoint proxy. See [Configure L7 logs with Istio Waypoint](configure-istio-waypoint.mdx). + +The table below details the key/value pairs in the JSON blob, including their [Elasticsearch datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html) and which collector(s) populate each field. "All" means every collector populates the field. A qualifier like "eBPF (TLS only)" means the field is populated only for events of that protocol family. + +:::note + +The `protocol` field (new with the eBPF collector) and the legacy `type` field both describe the wire protocol but use different value vocabularies. `protocol` is `http` or `tls`. `type` is `tcp`, `tls`, or `html/`. Filter on `protocol` for events from the eBPF collector and on `type` for events from Envoy. + +::: + +| Name | Datatype | Populated by | Description | +| ------------------------ | -------- | ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `host` | keyword | All | Name of the node that collected the L7 log entry. | +| `start_time` | date | All | Start time of log collection in Unix timestamp format. | +| `end_time` | date | All | End time of log collection in Unix timestamp format. | +| `bytes_in` | long | All | Number of incoming bytes since the last export. | +| `bytes_out` | long | All | Number of outgoing bytes since the last export. | +| `duration_mean` | long | All | Mean duration time of all the requests that match this combination of L7 data in nanoseconds. | +| `duration_max` | long | All | Max duration time of all the requests that match this combination of L7 data in nanoseconds. | +| `count` | long | All | Number of requests that match this combination of L7 data. | +| `src_name_aggr` | keyword | All | Contains one of the following values:
- Aggregated name of the source pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | +| `src_namespace` | keyword | All | Namespace of the source endpoint. | +| `src_type` | keyword | All | Source endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload's own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, the one with the longest-prefix match is chosen. Remaining ties are resolved alphabetically by the NetworkSet's full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | +| `dest_name_aggr` | keyword | All | Contains one of the following values:
- Aggregated name of the destination pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | +| `dest_namespace` | keyword | All | Namespace of the destination endpoint. | +| `dest_type` | keyword | All | Destination endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload's own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, CIDR matches outrank domain matches and longest-prefix wins between competing CIDR matches. Remaining ties are resolved alphabetically by the NetworkSet's full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | +| `dest_service_name` | keyword | All | Name of the destination service. This may be empty if the request was not made against a service. | +| `dest_service_namespace` | keyword | All | Namespace of the destination service. This may be empty if the request was not made against a service. | +| `dest_service_port` | long | All | Destination service port. | +| `url` | keyword | Envoy, eBPF (HTTP only) | URL that the request was made against. | +| `response_code` | keyword | Envoy, eBPF (HTTP only) | Response code returned by the request. | +| `method` | keyword | Envoy, eBPF (HTTP only) | HTTP method for the request. | +| `user_agent` | keyword | Envoy | User agent of the request. | +| `type` | keyword | Envoy | Type of request made. Possible values include `tcp`, `tls`, and `html/`. See the note above on `protocol` vs `type`. | +| `protocol` | keyword | eBPF | Wire-protocol family of the captured event. Possible values: `http`, `tls`. | +| `collector_name` | keyword | eBPF | Name of the collector that produced the event. For the kernel TCP-layer eBPF collector, the value is `ebpf-tcp`. | +| `tls_sni` | keyword | eBPF (TLS only) | TLS Server Name Indication sent by the client in the ClientHello. | +| `tls_version` | keyword | eBPF (TLS only) | Negotiated TLS protocol version (for example, `TLS 1.2`, `TLS 1.3`). | +| `tls_cipher_suite` | keyword | eBPF (TLS only) | Negotiated TLS cipher suite (for example, `TLS_AES_256_GCM_SHA384`). | diff --git a/sidebars-calico-enterprise.js b/sidebars-calico-enterprise.js index 5c9e7a3f73..84fffc49fd 100644 --- a/sidebars-calico-enterprise.js +++ b/sidebars-calico-enterprise.js @@ -402,7 +402,14 @@ module.exports = { type: 'category', label: 'L7 logs', link: { type: 'doc', id: 'observability/elastic/l7/index' }, - items: ['observability/elastic/l7/configure', 'observability/elastic/l7/datatypes'], + items: [ + 'observability/elastic/l7/configure', + 'observability/elastic/l7/configure-bpf', + 'observability/elastic/l7/configure-istio-waypoint', + 'observability/elastic/l7/aggregation', + 'observability/elastic/l7/datatypes', + 'observability/elastic/l7/datatypes-wip', + ], }, 'observability/elastic/troubleshoot', ], From dfbad7b14a16093f165c1dada7375b0b9f2a23eb Mon Sep 17 00:00:00 2001 From: Lucas Sampaio Date: Wed, 13 May 2026 12:27:27 -0700 Subject: [PATCH 2/3] Iterate on L7 aggregation sandbox and rename data types pages Aggregation sandbox redesign: - Group rows by aggregation bucket; expand/collapse to reveal merged events - Two-tone highlight (green for added/count up, red for removed/count down) - Color legend above the table; intro/footer condensed - More diverse sample data spanning cart, payments, and TLS scenarios Pages: - Promote the WIP data types page to datatypes.mdx; move the original to datatypes-legacy.mdx - Add collector_type and protocol_version fields - Sidebar reordered with "+ " prefix on the new pages and shorter labels (Configure with eBPF, Configure with Istio Waypoint, Aggregation, Data types) Co-Authored-By: Claude Opus 4.7 (1M context) --- .../components/L7AggregationSandbox/index.js | 267 +++++++++++++++--- .../observability/elastic/l7/aggregation.mdx | 14 +- .../elastic/l7/configure-bpf.mdx | 4 +- .../elastic/l7/datatypes-legacy.mdx | 36 +++ .../elastic/l7/datatypes-wip.mdx | 57 ---- .../observability/elastic/l7/datatypes.mdx | 65 +++-- sidebars-calico-enterprise.js | 10 +- 7 files changed, 311 insertions(+), 142 deletions(-) create mode 100644 calico-enterprise/observability/elastic/l7/datatypes-legacy.mdx delete mode 100644 calico-enterprise/observability/elastic/l7/datatypes-wip.mdx diff --git a/calico-enterprise/_includes/components/L7AggregationSandbox/index.js b/calico-enterprise/_includes/components/L7AggregationSandbox/index.js index 18379a124e..713ed3396f 100644 --- a/calico-enterprise/_includes/components/L7AggregationSandbox/index.js +++ b/calico-enterprise/_includes/components/L7AggregationSandbox/index.js @@ -1,16 +1,16 @@ -import React, { useState, useMemo } from 'react'; +import React, { useState, useMemo, useEffect, useRef } from 'react'; const SAMPLE = [ - { src: 'frontend', dest: 'cart', svc: 'cart', method: 'GET', url: '/cart', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, - { src: 'frontend', dest: 'cart', svc: 'cart', method: 'GET', url: '/cart', code: '200', ua: 'curl/8.0', protocol: 'http', sni: null }, - { src: 'frontend', dest: 'cart', svc: 'cart', method: 'PUT', url: '/cart', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, - { src: 'frontend', dest: 'cart', svc: 'cart', method: 'GET', url: '/products', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, - { src: 'frontend', dest: 'cart', svc: 'cart', method: 'GET', url: '/cart', code: '404', ua: 'cart-client/1.0', protocol: 'http', sni: null }, - { src: 'checkout', dest: 'cart', svc: 'cart', method: 'GET', url: '/cart', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, - { src: 'frontend', dest: 'cart-canary', svc: 'cart', method: 'GET', url: '/cart', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, - { src: 'frontend', dest: 'cart', svc: 'cart-v2', method: 'GET', url: '/cart', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, - { src: 'frontend', dest: 'cart', svc: 'cart', method: null, url: null, code: null, ua: null, protocol: 'tls', sni: 'cart.default.svc' }, - { src: 'frontend', dest: 'cart', svc: 'cart', method: null, url: null, code: null, ua: null, protocol: 'tls', sni: 'payment.default.svc' }, + { src: 'front', dest: 'cart', svc: 'cart-svc', method: 'GET', url: '/cart', code: '200', ua: 'client', protocol: 'http', sni: null }, + { src: 'front', dest: 'cart', svc: 'cart-svc', method: 'GET', url: '/cart', code: '404', ua: 'client', protocol: 'http', sni: null }, + { src: 'front', dest: 'cart', svc: 'cart-svc', method: 'GET', url: '/items', code: '200', ua: 'client', protocol: 'http', sni: null }, + { src: 'front', dest: 'cart', svc: 'cart-svc', method: 'POST', url: '/cart', code: '200', ua: 'client', protocol: 'http', sni: null }, + { src: 'orders', dest: 'pay', svc: 'pay-svc', method: 'POST', url: '/charge', code: '200', ua: 'sdk', protocol: 'http', sni: null }, + { src: 'edge', dest: 'pay', svc: 'pay-svc', method: 'POST', url: '/charge', code: '200', ua: 'sdk', protocol: 'http', sni: null }, + { src: 'orders', dest: 'canary', svc: 'pay-svc', method: 'POST', url: '/charge', code: '200', ua: 'sdk', protocol: 'http', sni: null }, + { src: 'orders', dest: 'pay', svc: 'pay-v2-svc', method: 'POST', url: '/charge', code: '200', ua: 'sdk', protocol: 'http', sni: null }, + { src: 'prom', dest: 'kibana', svc: 'kibana-svc', method: null, url: null, code: null, ua: null, protocol: 'tls', sni: 'kibana.svc' }, + { src: 'prom', dest: 'kibana', svc: 'kibana-svc', method: null, url: null, code: null, ua: null, protocol: 'tls', sni: 'es.svc' }, ]; const AGGREGATORS = [ @@ -36,50 +36,144 @@ const COLUMNS = [ { key: 'sni', label: 'tls_sni', owner: 'tlssni' }, ]; +const HIGHLIGHT_MS = 1500; + export default function L7AggregationSandbox() { const [checked, setChecked] = useState( Object.fromEntries(AGGREGATORS.map((a) => [a.id, true])), ); - const aggregated = useMemo(() => { - const buckets = new Map(); - for (const row of SAMPLE) { - const keyParts = [`protocol:${row.protocol}`]; + const displayedRows = useMemo(() => { + const bucketKeys = SAMPLE.map((row) => { + const parts = [`protocol:${row.protocol}`]; for (const a of AGGREGATORS) { if (!checked[a.id]) continue; - for (const f of a.fields) { - keyParts.push(`${f}:${row[f] ?? ''}`); - } + for (const f of a.fields) parts.push(`${f}:${row[f] ?? ''}`); } - const key = keyParts.join('|'); - if (buckets.has(key)) { - buckets.get(key).count += 1; - } else { - buckets.set(key, { ...row, count: 1 }); + return parts.join('|'); + }); + + const bucketCounts = new Map(); + const bucketOrder = new Map(); + let nextBucketIdx = 0; + for (const k of bucketKeys) { + bucketCounts.set(k, (bucketCounts.get(k) || 0) + 1); + if (!bucketOrder.has(k)) { + bucketOrder.set(k, nextBucketIdx++); } } - return Array.from(buckets.values()); + const primaryRowForBucket = new Map(); + + const rows = SAMPLE.map((row, originalIdx) => { + const k = bucketKeys[originalIdx]; + if (!primaryRowForBucket.has(k)) primaryRowForBucket.set(k, originalIdx); + const primaryOrigIdx = primaryRowForBucket.get(k); + const isPrimary = primaryOrigIdx === originalIdx; + const bucketIdx = bucketOrder.get(k); + const bucketSize = bucketCounts.get(k); + const out = { + originalIdx, + bucketIdx, + primaryOrigIdx, + bucketSize, + count: isPrimary ? bucketSize : '', + __primary: isPrimary, + }; + for (const col of COLUMNS) { + out[col.key] = + col.owner && !checked[col.owner] ? '—' : row[col.key] ?? '—'; + } + return out; + }); + + rows.sort((a, b) => { + if (a.bucketIdx !== b.bucketIdx) return a.bucketIdx - b.bucketIdx; + return a.originalIdx - b.originalIdx; + }); + + return rows; }, [checked]); - const toggle = (id) => - setChecked((c) => ({ ...c, [id]: !c[id] })); + const prevRef = useRef(displayedRows); + const [highlights, setHighlights] = useState(() => new Map()); + const [expanded, setExpanded] = useState(() => new Set()); - const cellValue = (row, col) => { - if (col.owner && !checked[col.owner]) return '—'; - return row[col.key] ?? '—'; + const toggleExpand = (originalIdx) => { + setExpanded((prev) => { + const next = new Set(prev); + if (next.has(originalIdx)) next.delete(originalIdx); + else next.add(originalIdx); + return next; + }); }; - const cellStyle = { + const visibleRows = displayedRows.filter( + (row) => row.__primary || expanded.has(row.primaryOrigIdx), + ); + + useEffect(() => { + const newHL = new Map(); + const prevByOrigIdx = new Map(); + for (const r of prevRef.current) { + prevByOrigIdx.set(r.originalIdx, r); + } + const HIGHLIGHTABLE = ['count', ...COLUMNS.map((c) => c.key)]; + const numOrZero = (v) => (typeof v === 'number' ? v : 0); + for (const row of displayedRows) { + const prev = prevByOrigIdx.get(row.originalIdx) || {}; + for (const k of HIGHLIGHTABLE) { + if (row[k] === prev[k]) continue; + let direction; + if (k === 'count') { + direction = numOrZero(row[k]) > numOrZero(prev[k]) ? 'add' : 'remove'; + } else { + direction = row[k] === '—' ? 'remove' : 'add'; + } + newHL.set(`${row.originalIdx}:${k}`, direction); + } + } + prevRef.current = displayedRows; + setHighlights(newHL); + if (newHL.size === 0) return; + const t = setTimeout(() => setHighlights(new Map()), HIGHLIGHT_MS); + return () => clearTimeout(t); + }, [displayedRows]); + + const toggle = (id) => setChecked((c) => ({ ...c, [id]: !c[id] })); + + const ROW_HEIGHT = '3.2em'; + const baseCellStyle = { padding: '4px 8px', fontFamily: 'var(--ifm-font-family-monospace)', fontSize: '0.85em', overflowWrap: 'anywhere', verticalAlign: 'top', + transition: 'background-color 1s ease', + height: ROW_HEIGHT, + boxSizing: 'border-box', + backgroundColor: 'transparent', + }; + const cellStyleFor = (rowIdx, key) => { + const direction = highlights.get(`${rowIdx}:${key}`); + if (direction === 'add') { + return { + ...baseCellStyle, + backgroundColor: 'var(--ifm-color-success-contrast-background)', + }; + } + if (direction === 'remove') { + return { + ...baseCellStyle, + backgroundColor: 'var(--ifm-color-danger-contrast-background)', + }; + } + return baseCellStyle; }; const headerStyle = { - ...cellStyle, + ...baseCellStyle, whiteSpace: 'normal', wordBreak: 'break-word', + transition: 'none', }; return ( @@ -114,37 +208,122 @@ export default function L7AggregationSandbox() { ))} +
+ + + added (value gained or count up) + + + + removed (value lost or count down) + +
+ {COLUMNS.map((c) => ( ))} - + {COLUMNS.map((c) => ( ))} - - {aggregated.map((row, i) => ( - - {COLUMNS.map((c) => ( - - ))} - - - ))} + {visibleRows.map((row, displayIdx) => { + const prevRow = visibleRows[displayIdx - 1]; + const isBucketStart = + !prevRow || prevRow.bucketIdx !== row.bucketIdx; + const borderTop = isBucketStart && displayIdx > 0 + ? '2px solid var(--ifm-color-emphasis-300)' + : undefined; + const styleForCell = (key) => { + const base = cellStyleFor(row.originalIdx, key); + return borderTop ? { ...base, borderTop } : base; + }; + const rowOpacity = row.__primary ? 1 : 0.7; + const isExpandable = row.__primary && row.bucketSize > 1; + const isOpen = expanded.has(row.originalIdx); + return ( + + + {COLUMNS.map((c) => ( + + ))} + + ); + })}
count{c.label}count
{cellValue(row, c)}{row.count}
+ {isExpandable ? ( + + ) : null} + {row.count} + {row[c.key]}
-

- Uncheck a field to drop it from the aggregation key. Rows that differ only in dropped fields merge, and the count column increments accordingly. -

); } diff --git a/calico-enterprise/observability/elastic/l7/aggregation.mdx b/calico-enterprise/observability/elastic/l7/aggregation.mdx index 431306c050..fcd76d6ec3 100644 --- a/calico-enterprise/observability/elastic/l7/aggregation.mdx +++ b/calico-enterprise/observability/elastic/l7/aggregation.mdx @@ -8,10 +8,16 @@ import L7AggregationSandbox from '../../../_includes/components/L7AggregationSan ## Big picture -$[prodname] aggregates L7 events before writing them to disk. Each row in the resulting log represents a unique combination of metadata fields and carries a `count` of how many requests matched that combination during the aggregation interval. Which fields participate in that combination — the aggregation key — is controlled by a set of fields on `FelixConfiguration`. +$[prodname] aggregates L7 events before writing them to disk. Each row in the resulting log represents a unique combination of metadata fields and carries a `count` of how many requests matched that combination during the aggregation interval. Which fields participate in that combination — the aggregation key — is controlled by a set of fields on [Felix Configuration](../../../reference/component-resources/node/felix/configuration.mdx#l7-logs). Including a field makes the logs more granular (more rows, fewer requests per row). Excluding a field collapses requests that differ only in that field into a single row and increments its `count` for each merged request. +## Aggregation playground + +The table below shows a small set of sample L7 events. Each visible row is what would be written to the L7 log, with `count` equal to the number of requests merged into it. Toggle the checkboxes to include or exclude each field from the aggregation key. + + + ## Aggregation fields The following `FelixConfiguration` fields control the L7 log aggregation key. For accepted values and defaults, follow the link to the [Felix Configuration reference](../../../reference/component-resources/node/felix/configuration.mdx#l7-logs). @@ -33,9 +39,3 @@ Two additional fields control how the URL is truncated before it enters the aggr | --- | --- | | [L7LogsFileAggregationNumURLPath](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationNumURLPath) | Maximum number of URL path components retained. Default `5`. A negative value disables this truncation. | | [L7LogsFileAggregationURLCharLimit](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationURLCharLimit) | Maximum URL length in characters; longer URLs are sliced. Default `250`. | - -## Aggregation playground - -The table below shows a small set of sample L7 events — eight HTTP requests and two TLS handshakes. Toggle the checkboxes to include or exclude each field from the aggregation key. The table will collapse and the `count` column will update to show how many original requests fall into each aggregated row. - - diff --git a/calico-enterprise/observability/elastic/l7/configure-bpf.mdx b/calico-enterprise/observability/elastic/l7/configure-bpf.mdx index c8ba965228..f0a190a13f 100644 --- a/calico-enterprise/observability/elastic/l7/configure-bpf.mdx +++ b/calico-enterprise/observability/elastic/l7/configure-bpf.mdx @@ -20,7 +20,7 @@ The eBPF-based collector delivers this visibility with no data-path proxy and no $[prodname] attaches eBPF probes to the kernel's TCP layer and extracts HTTP request/response and TLS handshake metadata from the traffic as it flows through the kernel. The parsed metadata is forwarded to the L7 log pipeline and written to disk on each node. -Each L7 log entry carries a `protocol` field (`http` or `tls`) and a `collector_name` field (`ebpf-tcp`) so events can be distinguished by their source. TLS entries additionally include `tls_sni`, `tls_version`, and `tls_cipher_suite`. For the full schema, see [L7 log data types](datatypes-wip.mdx). +Each L7 log entry carries a `protocol` field (`http` or `tls`) and a `collector_name` field (`ebpf-tcp`) so events can be distinguished by their source. TLS entries additionally include `tls_sni`, `tls_version`, and `tls_cipher_suite`. For the full schema, see [L7 log data types](datatypes.mdx). ### Comparison with the Istio Waypoint collector @@ -75,4 +75,4 @@ To tail the log on a node: tail -f /var/log/calico/l7logs/l7.log ``` -For the JSON schema, see [L7 log data types](datatypes-wip.mdx). +For the JSON schema, see [L7 log data types](datatypes.mdx). diff --git a/calico-enterprise/observability/elastic/l7/datatypes-legacy.mdx b/calico-enterprise/observability/elastic/l7/datatypes-legacy.mdx new file mode 100644 index 0000000000..478adeeb37 --- /dev/null +++ b/calico-enterprise/observability/elastic/l7/datatypes-legacy.mdx @@ -0,0 +1,36 @@ +--- +description: Reference of key/value fields that Calico Enterprise sends to Elasticsearch for L7 logs, including durations, byte counts, and HTTP request metadata. +--- + +# L7 log data types + +## Big picture + +$[prodname] sends the following data to Elasticsearch. + +The following table details the key/value pairs in the JSON blob, including their [Elasticsearch datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html). + +| Name | Datatype | Description | +| ------------------------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `host` | keyword | Name of the node that collected the L7 log entry. | +| `start_time` | date | Start time of log collection in Unix timestamp format. | +| `end_time` | date | End time of log collection in Unix timestamp format. | +| `bytes_in` | long | Number of incoming bytes since the last export. | +| `bytes_out` | long | Number of outgoing bytes since the last export. | +| `duration_mean` | long | Mean duration time of all the requests that match this combination of L7 data in nanoseconds. | +| `duration_max` | long | Max duration time of all the requests that match this combination of L7 data in nanoseconds. | +| `count` | long | Number of requests that match this combination of L7 data. | +| `src_name_aggr` | keyword | Contains one of the following values:
- Aggregated name of the source pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | +| `src_namespace` | keyword | Namespace of the source endpoint. | +| `src_type` | keyword | Source endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload’s own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, the one with the longest-prefix match is chosen. Remaining ties are resolved alphabetically by the NetworkSet’s full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | +| `dest_name_aggr` | keyword | Contains one of the following values:
- Aggregated name of the destination pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | +| `dest_namespace` | keyword | Namespace of the destination endpoint. | +| `dest_type` | keyword | Destination endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload’s own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, CIDR matches outrank domain matches and longest-prefix wins between competing CIDR matches. Remaining ties are resolved alphabetically by the NetworkSet’s full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | +| `dest_service_name` | keyword | Name of the destination service. This may be empty if the request was not made against a service. | +| `dest_service_namespace` | keyword | Namespace of the destination service. This may be empty if the request was not made against a service. | +| `dest_service_port` | long | Destination service port. | +| `url` | keyword | URL that the request was made against. | +| `response_code` | keyword | Response code returned by the request. | +| `method` | keyword | HTTP method for the request. | +| `user_agent` | keyword | User agent of the request. | +| `type` | keyword | Type of request made. Possible values include `tcp`, `tls`, and `html/`. | diff --git a/calico-enterprise/observability/elastic/l7/datatypes-wip.mdx b/calico-enterprise/observability/elastic/l7/datatypes-wip.mdx deleted file mode 100644 index e47fca38db..0000000000 --- a/calico-enterprise/observability/elastic/l7/datatypes-wip.mdx +++ /dev/null @@ -1,57 +0,0 @@ ---- -description: Reference of key/value fields that Calico Enterprise sends to Elasticsearch for L7 logs, with the collector that populates each field. ---- - -# L7 log data types (WIP) - -:::note - -This page is a work-in-progress redraft of [L7 log data types](datatypes.mdx). It adds the fields produced by the eBPF L7 collector and a **Populated by** column that identifies which collector fills each field. - -::: - -## Big picture - -$[prodname] supports multiple L7 collectors that write into the same Elasticsearch index (`tigera_secure_ee_l7`): - -- **Envoy** — the legacy sidecar/proxy-based collector. See [Configure L7 logs](configure.mdx). -- **eBPF** — kernel TCP-layer probes capturing HTTP request/response and TLS handshake metadata. See [Configure L7 logs with eBPF](configure-bpf.mdx). -- **Istio Waypoint** — collection through an Istio Ambient Mode waypoint proxy. See [Configure L7 logs with Istio Waypoint](configure-istio-waypoint.mdx). - -The table below details the key/value pairs in the JSON blob, including their [Elasticsearch datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html) and which collector(s) populate each field. "All" means every collector populates the field. A qualifier like "eBPF (TLS only)" means the field is populated only for events of that protocol family. - -:::note - -The `protocol` field (new with the eBPF collector) and the legacy `type` field both describe the wire protocol but use different value vocabularies. `protocol` is `http` or `tls`. `type` is `tcp`, `tls`, or `html/`. Filter on `protocol` for events from the eBPF collector and on `type` for events from Envoy. - -::: - -| Name | Datatype | Populated by | Description | -| ------------------------ | -------- | ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `host` | keyword | All | Name of the node that collected the L7 log entry. | -| `start_time` | date | All | Start time of log collection in Unix timestamp format. | -| `end_time` | date | All | End time of log collection in Unix timestamp format. | -| `bytes_in` | long | All | Number of incoming bytes since the last export. | -| `bytes_out` | long | All | Number of outgoing bytes since the last export. | -| `duration_mean` | long | All | Mean duration time of all the requests that match this combination of L7 data in nanoseconds. | -| `duration_max` | long | All | Max duration time of all the requests that match this combination of L7 data in nanoseconds. | -| `count` | long | All | Number of requests that match this combination of L7 data. | -| `src_name_aggr` | keyword | All | Contains one of the following values:
- Aggregated name of the source pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | -| `src_namespace` | keyword | All | Namespace of the source endpoint. | -| `src_type` | keyword | All | Source endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload's own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, the one with the longest-prefix match is chosen. Remaining ties are resolved alphabetically by the NetworkSet's full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | -| `dest_name_aggr` | keyword | All | Contains one of the following values:
- Aggregated name of the destination pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | -| `dest_namespace` | keyword | All | Namespace of the destination endpoint. | -| `dest_type` | keyword | All | Destination endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload's own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, CIDR matches outrank domain matches and longest-prefix wins between competing CIDR matches. Remaining ties are resolved alphabetically by the NetworkSet's full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | -| `dest_service_name` | keyword | All | Name of the destination service. This may be empty if the request was not made against a service. | -| `dest_service_namespace` | keyword | All | Namespace of the destination service. This may be empty if the request was not made against a service. | -| `dest_service_port` | long | All | Destination service port. | -| `url` | keyword | Envoy, eBPF (HTTP only) | URL that the request was made against. | -| `response_code` | keyword | Envoy, eBPF (HTTP only) | Response code returned by the request. | -| `method` | keyword | Envoy, eBPF (HTTP only) | HTTP method for the request. | -| `user_agent` | keyword | Envoy | User agent of the request. | -| `type` | keyword | Envoy | Type of request made. Possible values include `tcp`, `tls`, and `html/`. See the note above on `protocol` vs `type`. | -| `protocol` | keyword | eBPF | Wire-protocol family of the captured event. Possible values: `http`, `tls`. | -| `collector_name` | keyword | eBPF | Name of the collector that produced the event. For the kernel TCP-layer eBPF collector, the value is `ebpf-tcp`. | -| `tls_sni` | keyword | eBPF (TLS only) | TLS Server Name Indication sent by the client in the ClientHello. | -| `tls_version` | keyword | eBPF (TLS only) | Negotiated TLS protocol version (for example, `TLS 1.2`, `TLS 1.3`). | -| `tls_cipher_suite` | keyword | eBPF (TLS only) | Negotiated TLS cipher suite (for example, `TLS_AES_256_GCM_SHA384`). | diff --git a/calico-enterprise/observability/elastic/l7/datatypes.mdx b/calico-enterprise/observability/elastic/l7/datatypes.mdx index 478adeeb37..bcc8e969b4 100644 --- a/calico-enterprise/observability/elastic/l7/datatypes.mdx +++ b/calico-enterprise/observability/elastic/l7/datatypes.mdx @@ -1,36 +1,47 @@ --- -description: Reference of key/value fields that Calico Enterprise sends to Elasticsearch for L7 logs, including durations, byte counts, and HTTP request metadata. +description: Reference of key/value fields that Calico Enterprise sends to Elasticsearch for L7 logs, with the collector that populates each field. --- # L7 log data types ## Big picture -$[prodname] sends the following data to Elasticsearch. +$[prodname] supports multiple L7 collectors that write into the same Elasticsearch index: -The following table details the key/value pairs in the JSON blob, including their [Elasticsearch datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html). +- **Envoy** — the legacy sidecar/proxy-based collector. See [Configure L7 logs](configure.mdx). +- **eBPF** — kernel TCP-layer probes capturing HTTP request/response and TLS handshake metadata. See [Configure L7 logs with eBPF](configure-bpf.mdx). +- **Waypoint** — collection through an Istio Ambient Mode waypoint proxy. See [Configure L7 logs with Istio Waypoint](configure-istio-waypoint.mdx). -| Name | Datatype | Description | -| ------------------------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `host` | keyword | Name of the node that collected the L7 log entry. | -| `start_time` | date | Start time of log collection in Unix timestamp format. | -| `end_time` | date | End time of log collection in Unix timestamp format. | -| `bytes_in` | long | Number of incoming bytes since the last export. | -| `bytes_out` | long | Number of outgoing bytes since the last export. | -| `duration_mean` | long | Mean duration time of all the requests that match this combination of L7 data in nanoseconds. | -| `duration_max` | long | Max duration time of all the requests that match this combination of L7 data in nanoseconds. | -| `count` | long | Number of requests that match this combination of L7 data. | -| `src_name_aggr` | keyword | Contains one of the following values:
- Aggregated name of the source pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | -| `src_namespace` | keyword | Namespace of the source endpoint. | -| `src_type` | keyword | Source endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload’s own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, the one with the longest-prefix match is chosen. Remaining ties are resolved alphabetically by the NetworkSet’s full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | -| `dest_name_aggr` | keyword | Contains one of the following values:
- Aggregated name of the destination pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | -| `dest_namespace` | keyword | Namespace of the destination endpoint. | -| `dest_type` | keyword | Destination endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload’s own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, CIDR matches outrank domain matches and longest-prefix wins between competing CIDR matches. Remaining ties are resolved alphabetically by the NetworkSet’s full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | -| `dest_service_name` | keyword | Name of the destination service. This may be empty if the request was not made against a service. | -| `dest_service_namespace` | keyword | Namespace of the destination service. This may be empty if the request was not made against a service. | -| `dest_service_port` | long | Destination service port. | -| `url` | keyword | URL that the request was made against. | -| `response_code` | keyword | Response code returned by the request. | -| `method` | keyword | HTTP method for the request. | -| `user_agent` | keyword | User agent of the request. | -| `type` | keyword | Type of request made. Possible values include `tcp`, `tls`, and `html/`. | +The table below details the key/value pairs in the JSON blob, including their [Elasticsearch datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html) and which collector(s) populate each field. "All" means every collector populates the field. A qualifier like "eBPF (TLS only)" means the field is populated only for events of that protocol family. + +| Name | Datatype | Populated by | Description | +| ------------------------ | -------- | ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `host` | keyword | All | Name of the node that collected the L7 log entry. | +| `start_time` | date | All | Start time of log collection in Unix timestamp format. | +| `end_time` | date | All | End time of log collection in Unix timestamp format. | +| `bytes_in` | long | All | Number of incoming bytes since the last export. | +| `bytes_out` | long | All | Number of outgoing bytes since the last export. | +| `duration_mean` | long | All | Mean duration time of all the requests that match this combination of L7 data in nanoseconds. | +| `duration_max` | long | All | Max duration time of all the requests that match this combination of L7 data in nanoseconds. | +| `count` | long | All | Number of requests that match this combination of L7 data. | +| `src_name_aggr` | keyword | All | Contains one of the following values:
- Aggregated name of the source pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | +| `src_namespace` | keyword | All | Namespace of the source endpoint. | +| `src_type` | keyword | All | Source endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload's own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, the one with the longest-prefix match is chosen. Remaining ties are resolved alphabetically by the NetworkSet's full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | +| `dest_name_aggr` | keyword | All | Contains one of the following values:
- Aggregated name of the destination pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | +| `dest_namespace` | keyword | All | Namespace of the destination endpoint. | +| `dest_type` | keyword | All | Destination endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload's own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, CIDR matches outrank domain matches and longest-prefix wins between competing CIDR matches. Remaining ties are resolved alphabetically by the NetworkSet's full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | +| `dest_service_name` | keyword | All | Name of the destination service. This may be empty if the request was not made against a service. | +| `dest_service_namespace` | keyword | All | Namespace of the destination service. This may be empty if the request was not made against a service. | +| `dest_service_port` | long | All | Destination service port. | +| `url` | keyword | All (HTTP only) | URL that the request was made against. | +| `response_code` | keyword | All (HTTP only) | Response code returned by the request. | +| `method` | keyword | All (HTTP only) | HTTP method for the request. | +| `user_agent` | keyword | Envoy, Waypoint | User agent of the request. | +| `type` | keyword | Envoy, Waypoint | Type of request made. Possible values include `tcp`, `tls`, and `html/`. | +| `protocol` | keyword | eBPF, Waypoint | Wire-protocol family of the captured event. Possible values: `http`, `tls`. | +| `protocol_version` | keyword | eBPF, Waypoint | Wire-protocol version. For HTTP events, possible values include `1.0` and `1.1`. | +| `collector_type` | keyword | eBPF, Waypoint | Category of the collector that produced the event. Possible values include `ebpf` and `waypoint`. | +| `collector_name` | keyword | eBPF, Waypoint | Name of the specific collector that produced the event. For the kernel TCP-layer eBPF collector, the value is `ebpf-tcp`. | +| `tls_sni` | keyword | eBPF (TLS only) | TLS Server Name Indication sent by the client in the ClientHello. | +| `tls_version` | keyword | eBPF (TLS only) | Negotiated TLS protocol version (for example, `TLS 1.2`, `TLS 1.3`). | +| `tls_cipher_suite` | keyword | eBPF (TLS only) | Negotiated TLS cipher suite (for example, `TLS_AES_256_GCM_SHA384`). | diff --git a/sidebars-calico-enterprise.js b/sidebars-calico-enterprise.js index 84fffc49fd..b5835dbc7a 100644 --- a/sidebars-calico-enterprise.js +++ b/sidebars-calico-enterprise.js @@ -403,12 +403,12 @@ module.exports = { label: 'L7 logs', link: { type: 'doc', id: 'observability/elastic/l7/index' }, items: [ + { type: 'doc', id: 'observability/elastic/l7/configure-bpf', label: '+ Configure with eBPF' }, + { type: 'doc', id: 'observability/elastic/l7/configure-istio-waypoint', label: '+ Configure with Istio Waypoint' }, + { type: 'doc', id: 'observability/elastic/l7/datatypes', label: '+ Data types' }, + { type: 'doc', id: 'observability/elastic/l7/aggregation', label: '+ Aggregation' }, 'observability/elastic/l7/configure', - 'observability/elastic/l7/configure-bpf', - 'observability/elastic/l7/configure-istio-waypoint', - 'observability/elastic/l7/aggregation', - 'observability/elastic/l7/datatypes', - 'observability/elastic/l7/datatypes-wip', + { type: 'doc', id: 'observability/elastic/l7/datatypes-legacy', label: 'L7 log data types' }, ], }, 'observability/elastic/troubleshoot', From 1eef8f32234f84b89ead5e92183b1aea9002e302 Mon Sep 17 00:00:00 2001 From: Lucas Sampaio Date: Wed, 13 May 2026 13:28:26 -0700 Subject: [PATCH 3/3] Fix Vale lint errors on new L7 pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - dataplane / dataplanes → data plane / data planes - waypoint → Waypoint (Calico term rule) - Plaintext / plaintext → Plain text / plain text - Rephrase HTTP pipelining bullet to drop dictionary-rejected words Co-Authored-By: Claude Opus 4.7 (1M context) --- .../observability/elastic/l7/configure-bpf.mdx | 14 +++++++------- .../elastic/l7/configure-istio-waypoint.mdx | 4 ++-- .../observability/elastic/l7/datatypes.mdx | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/calico-enterprise/observability/elastic/l7/configure-bpf.mdx b/calico-enterprise/observability/elastic/l7/configure-bpf.mdx index f0a190a13f..35febe09bf 100644 --- a/calico-enterprise/observability/elastic/l7/configure-bpf.mdx +++ b/calico-enterprise/observability/elastic/l7/configure-bpf.mdx @@ -12,7 +12,7 @@ Use $[prodname] eBPF-based L7 observability to capture HTTP request/response and L7 logs give platform operators and development teams visibility into how applications talk to each other beyond what L3/4 flow logs show: HTTP method, URL, response code, and TLS SNI / version / cipher suite. They are also key for spotting anomalous behavior such as attempts to access restricted URLs or scans for particular paths. -The eBPF-based collector delivers this visibility with no data-path proxy and no per-deployment opt-in: enable it once at the cluster level, and HTTP and TLS metadata is captured for every workload on the node. The feature is **dataplane-independent** and works with the eBPF, iptables, and nftables dataplanes alike. +The eBPF-based collector delivers this visibility with no data-path proxy and no per-deployment opt-in: enable it once at the cluster level, and HTTP and TLS metadata is captured for every workload on the node. The feature is **data-plane-independent** and works with the eBPF, iptables, and nftables data planes alike. ## Concepts @@ -24,14 +24,14 @@ Each L7 log entry carries a `protocol` field (`http` or `tls`) and a `collector_ ### Comparison with the Istio Waypoint collector -The [Istio Waypoint collector](configure-istio-waypoint.mdx) routes selected traffic through a per-namespace Envoy waypoint proxy and produces L7 logs from the proxy itself. The eBPF collector runs inside Felix on every node and captures HTTP and TLS metadata as traffic passes through the kernel TCP layer. The two collectors can coexist in the same cluster; both feed the same L7 log pipeline and entries are distinguished by the `collector_name` field. +The [Istio Waypoint collector](configure-istio-waypoint.mdx) routes selected traffic through a per-namespace Envoy Waypoint proxy and produces L7 logs from the proxy itself. The eBPF collector runs inside Felix on every node and captures HTTP and TLS metadata as traffic passes through the kernel TCP layer. The two collectors can coexist in the same cluster; both feed the same L7 log pipeline and entries are distinguished by the `collector_name` field. | Aspect | Istio Waypoint collector | eBPF L7 collector | | --- | --- | --- | | Additional infrastructure | Waypoint Envoy pods per namespace/service | None — runs inside Felix | -| L7 coverage | Only traffic routed through a waypoint | All TCP traffic on the node | +| L7 coverage | Only traffic routed through a Waypoint | All TCP traffic on the node | | Network path overhead | Two extra proxy hops | None | -| Plaintext access | Decrypted at the waypoint (mTLS terminated) | Captured at the kernel TCP layer, before TLS encryption | +| Plain text access | Decrypted at the Waypoint (mTLS terminated) | Captured at the kernel TCP layer, before TLS encryption | | Protocol coverage | HTTP/1, HTTP/2, gRPC | HTTP/1.0, HTTP/1.1, TLS handshake metadata | | Opt-in granularity | Per namespace/service | Cluster-wide via FelixConfiguration | | L7 features beyond observability | Retries, traffic splitting, fault injection | Observability only | @@ -41,8 +41,8 @@ The [Istio Waypoint collector](configure-istio-waypoint.mdx) routes selected tra ### Limitations * Requires Linux kernel 5.17 or later on every node. -* HTTP parsing covers HTTP/1.0 and HTTP/1.1 only, because their requests and responses travel as plaintext on the wire. HTTP/2 and HTTP/3 are not captured as HTTP, and encrypted HTTP traffic surfaces only as TLS metadata. -* HTTP captures only the most recent request per socket; rapidly pipelined requests on a single connection may be undercounted. +* HTTP parsing covers HTTP/1.0 and HTTP/1.1 only, because their requests and responses travel as plain text on the wire. HTTP/2 and HTTP/3 are not captured as HTTP, and encrypted HTTP traffic surfaces only as TLS metadata. +* HTTP capture tracks only the most recent request per socket; when a single connection carries multiple back-to-back requests, some may be missed. * TLS capture is limited to handshake metadata (SNI, version, cipher suite). The collector does not decrypt application traffic. :::note @@ -53,7 +53,7 @@ L7 logs require a minimum of 1 additional GB of log storage per node, per one-da ## Configure L7 logs -Enable the eBPF L7 collector by setting the [L7ObservabilityEnabled](../../../reference/component-resources/node/felix/configuration.mdx#L7ObservabilityEnabled) field to `true` on the default `FelixConfiguration`. The setting is independent of `bpfEnabled` — the collector works with any dataplane. +Enable the eBPF L7 collector by setting the [L7ObservabilityEnabled](../../../reference/component-resources/node/felix/configuration.mdx#L7ObservabilityEnabled) field to `true` on the default `FelixConfiguration`. The setting is independent of `bpfEnabled` — the collector works with any data plane. ```bash kubectl patch felixconfiguration default --type=merge -p '{"spec":{"l7ObservabilityEnabled":true}}' diff --git a/calico-enterprise/observability/elastic/l7/configure-istio-waypoint.mdx b/calico-enterprise/observability/elastic/l7/configure-istio-waypoint.mdx index 69a13ef22e..173fe8a852 100644 --- a/calico-enterprise/observability/elastic/l7/configure-istio-waypoint.mdx +++ b/calico-enterprise/observability/elastic/l7/configure-istio-waypoint.mdx @@ -1,12 +1,12 @@ --- -description: Capture L7 logs through an Istio Ambient Mode waypoint proxy. +description: Capture L7 logs through an Istio Ambient Mode Waypoint proxy. --- # Configure L7 logs with Istio Waypoint :::note -This page is a placeholder. Content for L7 log collection via Istio Ambient Mode waypoint proxies is pending. +This page is a placeholder. Content for L7 log collection via Istio Ambient Mode Waypoint proxies is pending. ::: diff --git a/calico-enterprise/observability/elastic/l7/datatypes.mdx b/calico-enterprise/observability/elastic/l7/datatypes.mdx index bcc8e969b4..1e98127fa1 100644 --- a/calico-enterprise/observability/elastic/l7/datatypes.mdx +++ b/calico-enterprise/observability/elastic/l7/datatypes.mdx @@ -10,7 +10,7 @@ $[prodname] supports multiple L7 collectors that write into the same Elasticsear - **Envoy** — the legacy sidecar/proxy-based collector. See [Configure L7 logs](configure.mdx). - **eBPF** — kernel TCP-layer probes capturing HTTP request/response and TLS handshake metadata. See [Configure L7 logs with eBPF](configure-bpf.mdx). -- **Waypoint** — collection through an Istio Ambient Mode waypoint proxy. See [Configure L7 logs with Istio Waypoint](configure-istio-waypoint.mdx). +- **Waypoint** — collection through an Istio Ambient Mode Waypoint proxy. See [Configure L7 logs with Istio Waypoint](configure-istio-waypoint.mdx). The table below details the key/value pairs in the JSON blob, including their [Elasticsearch datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html) and which collector(s) populate each field. "All" means every collector populates the field. A qualifier like "eBPF (TLS only)" means the field is populated only for events of that protocol family.