diff --git a/calico-enterprise/_includes/components/L7AggregationSandbox/index.js b/calico-enterprise/_includes/components/L7AggregationSandbox/index.js new file mode 100644 index 0000000000..713ed3396f --- /dev/null +++ b/calico-enterprise/_includes/components/L7AggregationSandbox/index.js @@ -0,0 +1,329 @@ +import React, { useState, useMemo, useEffect, useRef } from 'react'; + +const SAMPLE = [ + { src: 'front', dest: 'cart', svc: 'cart-svc', method: 'GET', url: '/cart', code: '200', ua: 'client', protocol: 'http', sni: null }, + { src: 'front', dest: 'cart', svc: 'cart-svc', method: 'GET', url: '/cart', code: '404', ua: 'client', protocol: 'http', sni: null }, + { src: 'front', dest: 'cart', svc: 'cart-svc', method: 'GET', url: '/items', code: '200', ua: 'client', protocol: 'http', sni: null }, + { src: 'front', dest: 'cart', svc: 'cart-svc', method: 'POST', url: '/cart', code: '200', ua: 'client', protocol: 'http', sni: null }, + { src: 'orders', dest: 'pay', svc: 'pay-svc', method: 'POST', url: '/charge', code: '200', ua: 'sdk', protocol: 'http', sni: null }, + { src: 'edge', dest: 'pay', svc: 'pay-svc', method: 'POST', url: '/charge', code: '200', ua: 'sdk', protocol: 'http', sni: null }, + { src: 'orders', dest: 'canary', svc: 'pay-svc', method: 'POST', url: '/charge', code: '200', ua: 'sdk', protocol: 'http', sni: null }, + { src: 'orders', dest: 'pay', svc: 'pay-v2-svc', method: 'POST', url: '/charge', code: '200', ua: 'sdk', protocol: 'http', sni: null }, + { src: 'prom', dest: 'kibana', svc: 'kibana-svc', method: null, url: null, code: null, ua: null, protocol: 'tls', sni: 'kibana.svc' }, + { src: 'prom', dest: 'kibana', svc: 'kibana-svc', method: null, url: null, code: null, ua: null, protocol: 'tls', sni: 'es.svc' }, +]; + +const AGGREGATORS = [ + { id: 'source', label: 'Source info', fields: ['src'], field: 'L7LogsFileAggregationSourceInfo' }, + { id: 'dest', label: 'Destination info', fields: ['dest'], field: 'L7LogsFileAggregationDestinationInfo' }, + { id: 'service', label: 'Service info', fields: ['svc'], field: 'L7LogsFileAggregationServiceInfo' }, + { id: 'method', label: 'HTTP method', fields: ['method'], field: 'L7LogsFileAggregationHTTPMethod' }, + { id: 'url', label: 'URL', fields: ['url'], field: 'L7LogsFileAggregationTrimURL' }, + { id: 'code', label: 'Response code', fields: ['code'], field: 'L7LogsFileAggregationResponseCode' }, + { id: 'header', label: 'HTTP header info', fields: ['ua'], field: 'L7LogsFileAggregationHTTPHeaderInfo' }, + { id: 'tlssni', label: 'TLS SNI', fields: ['sni'], field: 'L7LogsFileAggregationTLSSNI' }, +]; + +const COLUMNS = [ + { key: 'protocol', label: 'protocol', owner: null }, + { key: 'src', label: 'src_name_aggr', owner: 'source' }, + { key: 'dest', label: 'dest_name_aggr', owner: 'dest' }, + { key: 'svc', label: 'dest_service_name', owner: 'service' }, + { key: 'method', label: 'method', owner: 'method' }, + { key: 'url', label: 'url', owner: 'url' }, + { key: 'code', label: 'response_code', owner: 'code' }, + { key: 'ua', label: 'user_agent', owner: 'header' }, + { key: 'sni', label: 'tls_sni', owner: 'tlssni' }, +]; + +const HIGHLIGHT_MS = 1500; + +export default function L7AggregationSandbox() { + const [checked, setChecked] = useState( + Object.fromEntries(AGGREGATORS.map((a) => [a.id, true])), + ); + + const displayedRows = useMemo(() => { + const bucketKeys = SAMPLE.map((row) => { + const parts = [`protocol:${row.protocol}`]; + for (const a of AGGREGATORS) { + if (!checked[a.id]) continue; + for (const f of a.fields) parts.push(`${f}:${row[f] ?? ''}`); + } + return parts.join('|'); + }); + + const bucketCounts = new Map(); + const bucketOrder = new Map(); + let nextBucketIdx = 0; + for (const k of bucketKeys) { + bucketCounts.set(k, (bucketCounts.get(k) || 0) + 1); + if (!bucketOrder.has(k)) { + bucketOrder.set(k, nextBucketIdx++); + } + } + const primaryRowForBucket = new Map(); + + const rows = SAMPLE.map((row, originalIdx) => { + const k = bucketKeys[originalIdx]; + if (!primaryRowForBucket.has(k)) primaryRowForBucket.set(k, originalIdx); + const primaryOrigIdx = primaryRowForBucket.get(k); + const isPrimary = primaryOrigIdx === originalIdx; + const bucketIdx = bucketOrder.get(k); + const bucketSize = bucketCounts.get(k); + const out = { + originalIdx, + bucketIdx, + primaryOrigIdx, + bucketSize, + count: isPrimary ? bucketSize : '', + __primary: isPrimary, + }; + for (const col of COLUMNS) { + out[col.key] = + col.owner && !checked[col.owner] ? '—' : row[col.key] ?? '—'; + } + return out; + }); + + rows.sort((a, b) => { + if (a.bucketIdx !== b.bucketIdx) return a.bucketIdx - b.bucketIdx; + return a.originalIdx - b.originalIdx; + }); + + return rows; + }, [checked]); + + const prevRef = useRef(displayedRows); + const [highlights, setHighlights] = useState(() => new Map()); + const [expanded, setExpanded] = useState(() => new Set()); + + const toggleExpand = (originalIdx) => { + setExpanded((prev) => { + const next = new Set(prev); + if (next.has(originalIdx)) next.delete(originalIdx); + else next.add(originalIdx); + return next; + }); + }; + + const visibleRows = displayedRows.filter( + (row) => row.__primary || expanded.has(row.primaryOrigIdx), + ); + + useEffect(() => { + const newHL = new Map(); + const prevByOrigIdx = new Map(); + for (const r of prevRef.current) { + prevByOrigIdx.set(r.originalIdx, r); + } + const HIGHLIGHTABLE = ['count', ...COLUMNS.map((c) => c.key)]; + const numOrZero = (v) => (typeof v === 'number' ? v : 0); + for (const row of displayedRows) { + const prev = prevByOrigIdx.get(row.originalIdx) || {}; + for (const k of HIGHLIGHTABLE) { + if (row[k] === prev[k]) continue; + let direction; + if (k === 'count') { + direction = numOrZero(row[k]) > numOrZero(prev[k]) ? 'add' : 'remove'; + } else { + direction = row[k] === '—' ? 'remove' : 'add'; + } + newHL.set(`${row.originalIdx}:${k}`, direction); + } + } + prevRef.current = displayedRows; + setHighlights(newHL); + if (newHL.size === 0) return; + const t = setTimeout(() => setHighlights(new Map()), HIGHLIGHT_MS); + return () => clearTimeout(t); + }, [displayedRows]); + + const toggle = (id) => setChecked((c) => ({ ...c, [id]: !c[id] })); + + const ROW_HEIGHT = '3.2em'; + const baseCellStyle = { + padding: '4px 8px', + fontFamily: 'var(--ifm-font-family-monospace)', + fontSize: '0.85em', + overflowWrap: 'anywhere', + verticalAlign: 'top', + transition: 'background-color 1s ease', + height: ROW_HEIGHT, + boxSizing: 'border-box', + backgroundColor: 'transparent', + }; + const cellStyleFor = (rowIdx, key) => { + const direction = highlights.get(`${rowIdx}:${key}`); + if (direction === 'add') { + return { + ...baseCellStyle, + backgroundColor: 'var(--ifm-color-success-contrast-background)', + }; + } + if (direction === 'remove') { + return { + ...baseCellStyle, + backgroundColor: 'var(--ifm-color-danger-contrast-background)', + }; + } + return baseCellStyle; + }; + const headerStyle = { + ...baseCellStyle, + whiteSpace: 'normal', + wordBreak: 'break-word', + transition: 'none', + }; + + return ( +
+
+ {AGGREGATORS.map((a) => ( + + ))} +
+
+ + + added (value gained or count up) + + + + removed (value lost or count down) + +
+
+ + + + {COLUMNS.map((c) => ( + + ))} + + + + + {COLUMNS.map((c) => ( + + ))} + + + + {visibleRows.map((row, displayIdx) => { + const prevRow = visibleRows[displayIdx - 1]; + const isBucketStart = + !prevRow || prevRow.bucketIdx !== row.bucketIdx; + const borderTop = isBucketStart && displayIdx > 0 + ? '2px solid var(--ifm-color-emphasis-300)' + : undefined; + const styleForCell = (key) => { + const base = cellStyleFor(row.originalIdx, key); + return borderTop ? { ...base, borderTop } : base; + }; + const rowOpacity = row.__primary ? 1 : 0.7; + const isExpandable = row.__primary && row.bucketSize > 1; + const isOpen = expanded.has(row.originalIdx); + return ( + + + {COLUMNS.map((c) => ( + + ))} + + ); + })} + +
count{c.label}
+ {isExpandable ? ( + + ) : null} + {row.count} + {row[c.key]}
+
+
+ ); +} diff --git a/calico-enterprise/observability/elastic/l7/aggregation.mdx b/calico-enterprise/observability/elastic/l7/aggregation.mdx new file mode 100644 index 0000000000..fcd76d6ec3 --- /dev/null +++ b/calico-enterprise/observability/elastic/l7/aggregation.mdx @@ -0,0 +1,41 @@ +--- +description: Control how Calico Enterprise aggregates L7 log entries by selecting which fields participate in the aggregation key. +--- + +import L7AggregationSandbox from '../../../_includes/components/L7AggregationSandbox'; + +# L7 log aggregation + +## Big picture + +$[prodname] aggregates L7 events before writing them to disk. Each row in the resulting log represents a unique combination of metadata fields and carries a `count` of how many requests matched that combination during the aggregation interval. Which fields participate in that combination — the aggregation key — is controlled by a set of fields on [Felix Configuration](../../../reference/component-resources/node/felix/configuration.mdx#l7-logs). + +Including a field makes the logs more granular (more rows, fewer requests per row). Excluding a field collapses requests that differ only in that field into a single row and increments its `count` for each merged request. + +## Aggregation playground + +The table below shows a small set of sample L7 events. Each visible row is what would be written to the L7 log, with `count` equal to the number of requests merged into it. Toggle the checkboxes to include or exclude each field from the aggregation key. + + + +## Aggregation fields + +The following `FelixConfiguration` fields control the L7 log aggregation key. For accepted values and defaults, follow the link to the [Felix Configuration reference](../../../reference/component-resources/node/felix/configuration.mdx#l7-logs). + +| Field | Effect when included | Effect when excluded | +| --- | --- | --- | +| [L7LogsFileAggregationSourceInfo](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationSourceInfo) | Source aggregated name, namespace, and type appear on each row. | Requests are merged across sources. | +| [L7LogsFileAggregationDestinationInfo](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationDestinationInfo) | Destination aggregated name, namespace, and type appear on each row. | Requests are merged across destinations. | +| [L7LogsFileAggregationServiceInfo](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationServiceInfo) | Destination service name, namespace, and port appear on each row. | Requests are merged across services. | +| [L7LogsFileAggregationHTTPMethod](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationHTTPMethod) | HTTP method appears on each row. | Requests are merged across methods. | +| [L7LogsFileAggregationTrimURL](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationTrimURL) | URL (or a trimmed prefix) appears on each row. Intermediate values trim the query string, then the path, before excluding the URL entirely. | Requests are merged across URLs. | +| [L7LogsFileAggregationResponseCode](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationResponseCode) | Response code appears on each row. | Requests are merged across response codes. | +| [L7LogsFileAggregationHTTPHeaderInfo](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationHTTPHeaderInfo) | User agent and request type appear on each row. | Requests are merged across user agents and types. | +| [L7LogsFileAggregationTLSSNI](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationTLSSNI) | TLS Server Name Indication appears on each TLS row. | TLS flows to the same destination are merged regardless of SNI. | + +Two additional fields control how the URL is truncated before it enters the aggregation key, rather than whether the URL participates at all: + +| Field | Description | +| --- | --- | +| [L7LogsFileAggregationNumURLPath](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationNumURLPath) | Maximum number of URL path components retained. Default `5`. A negative value disables this truncation. | +| [L7LogsFileAggregationURLCharLimit](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationURLCharLimit) | Maximum URL length in characters; longer URLs are sliced. Default `250`. | diff --git a/calico-enterprise/observability/elastic/l7/configure-bpf.mdx b/calico-enterprise/observability/elastic/l7/configure-bpf.mdx new file mode 100644 index 0000000000..35febe09bf --- /dev/null +++ b/calico-enterprise/observability/elastic/l7/configure-bpf.mdx @@ -0,0 +1,78 @@ +--- +description: Capture HTTP and TLS L7 logs from kernel TCP probes using eBPF, with no proxy or sidecar injection required. +--- + +# Configure L7 logs with eBPF + +## Big picture + +Use $[prodname] eBPF-based L7 observability to capture HTTP request/response and TLS handshake metadata directly from the kernel TCP layer, without deploying a proxy or injecting sidecars. + +## Value + +L7 logs give platform operators and development teams visibility into how applications talk to each other beyond what L3/4 flow logs show: HTTP method, URL, response code, and TLS SNI / version / cipher suite. They are also key for spotting anomalous behavior such as attempts to access restricted URLs or scans for particular paths. + +The eBPF-based collector delivers this visibility with no data-path proxy and no per-deployment opt-in: enable it once at the cluster level, and HTTP and TLS metadata is captured for every workload on the node. The feature is **data-plane-independent** and works with the eBPF, iptables, and nftables data planes alike. + +## Concepts + +### About eBPF L7 logs + +$[prodname] attaches eBPF probes to the kernel's TCP layer and extracts HTTP request/response and TLS handshake metadata from the traffic as it flows through the kernel. The parsed metadata is forwarded to the L7 log pipeline and written to disk on each node. + +Each L7 log entry carries a `protocol` field (`http` or `tls`) and a `collector_name` field (`ebpf-tcp`) so events can be distinguished by their source. TLS entries additionally include `tls_sni`, `tls_version`, and `tls_cipher_suite`. For the full schema, see [L7 log data types](datatypes.mdx). + +### Comparison with the Istio Waypoint collector + +The [Istio Waypoint collector](configure-istio-waypoint.mdx) routes selected traffic through a per-namespace Envoy Waypoint proxy and produces L7 logs from the proxy itself. The eBPF collector runs inside Felix on every node and captures HTTP and TLS metadata as traffic passes through the kernel TCP layer. The two collectors can coexist in the same cluster; both feed the same L7 log pipeline and entries are distinguished by the `collector_name` field. + +| Aspect | Istio Waypoint collector | eBPF L7 collector | +| --- | --- | --- | +| Additional infrastructure | Waypoint Envoy pods per namespace/service | None — runs inside Felix | +| L7 coverage | Only traffic routed through a Waypoint | All TCP traffic on the node | +| Network path overhead | Two extra proxy hops | None | +| Plain text access | Decrypted at the Waypoint (mTLS terminated) | Captured at the kernel TCP layer, before TLS encryption | +| Protocol coverage | HTTP/1, HTTP/2, gRPC | HTTP/1.0, HTTP/1.1, TLS handshake metadata | +| Opt-in granularity | Per namespace/service | Cluster-wide via FelixConfiguration | +| L7 features beyond observability | Retries, traffic splitting, fault injection | Observability only | + +## Before you begin + +### Limitations + +* Requires Linux kernel 5.17 or later on every node. +* HTTP parsing covers HTTP/1.0 and HTTP/1.1 only, because their requests and responses travel as plain text on the wire. HTTP/2 and HTTP/3 are not captured as HTTP, and encrypted HTTP traffic surfaces only as TLS metadata. +* HTTP capture tracks only the most recent request per socket; when a single connection carries multiple back-to-back requests, some may be missed. +* TLS capture is limited to handshake metadata (SNI, version, cipher suite). The collector does not decrypt application traffic. + +:::note + +L7 logs require a minimum of 1 additional GB of log storage per node, per one-day retention period. Adjust your [Log Storage](../../../operations/logstorage/adjust-log-storage-size.mdx) before you start tasks in the next section. + +::: + +## Configure L7 logs + +Enable the eBPF L7 collector by setting the [L7ObservabilityEnabled](../../../reference/component-resources/node/felix/configuration.mdx#L7ObservabilityEnabled) field to `true` on the default `FelixConfiguration`. The setting is independent of `bpfEnabled` — the collector works with any data plane. + +```bash +kubectl patch felixconfiguration default --type=merge -p '{"spec":{"l7ObservabilityEnabled":true}}' +``` + +To disable the collector, set the field back to `false`. Felix tears down its BPF programs and cleans up pinned maps; no node reboot is required. + +```bash +kubectl patch felixconfiguration default --type=merge -p '{"spec":{"l7ObservabilityEnabled":false}}' +``` + +## View L7 logs + +The eBPF collector writes L7 events as JSON to a log file on each node. By default the file is at `/var/log/calico/l7logs/l7.log`. The directory is controlled by the [L7LogsFileDirectory](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileDirectory) field on `FelixConfiguration`. + +To tail the log on a node: + +```bash +tail -f /var/log/calico/l7logs/l7.log +``` + +For the JSON schema, see [L7 log data types](datatypes.mdx). diff --git a/calico-enterprise/observability/elastic/l7/configure-istio-waypoint.mdx b/calico-enterprise/observability/elastic/l7/configure-istio-waypoint.mdx new file mode 100644 index 0000000000..173fe8a852 --- /dev/null +++ b/calico-enterprise/observability/elastic/l7/configure-istio-waypoint.mdx @@ -0,0 +1,23 @@ +--- +description: Capture L7 logs through an Istio Ambient Mode Waypoint proxy. +--- + +# Configure L7 logs with Istio Waypoint + +:::note + +This page is a placeholder. Content for L7 log collection via Istio Ambient Mode Waypoint proxies is pending. + +::: + +## Big picture + +## Value + +## Concepts + +## Before you begin + +## Configure L7 logs + +## View L7 logs in the web console diff --git a/calico-enterprise/observability/elastic/l7/datatypes-legacy.mdx b/calico-enterprise/observability/elastic/l7/datatypes-legacy.mdx new file mode 100644 index 0000000000..478adeeb37 --- /dev/null +++ b/calico-enterprise/observability/elastic/l7/datatypes-legacy.mdx @@ -0,0 +1,36 @@ +--- +description: Reference of key/value fields that Calico Enterprise sends to Elasticsearch for L7 logs, including durations, byte counts, and HTTP request metadata. +--- + +# L7 log data types + +## Big picture + +$[prodname] sends the following data to Elasticsearch. + +The following table details the key/value pairs in the JSON blob, including their [Elasticsearch datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html). + +| Name | Datatype | Description | +| ------------------------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `host` | keyword | Name of the node that collected the L7 log entry. | +| `start_time` | date | Start time of log collection in Unix timestamp format. | +| `end_time` | date | End time of log collection in Unix timestamp format. | +| `bytes_in` | long | Number of incoming bytes since the last export. | +| `bytes_out` | long | Number of outgoing bytes since the last export. | +| `duration_mean` | long | Mean duration time of all the requests that match this combination of L7 data in nanoseconds. | +| `duration_max` | long | Max duration time of all the requests that match this combination of L7 data in nanoseconds. | +| `count` | long | Number of requests that match this combination of L7 data. | +| `src_name_aggr` | keyword | Contains one of the following values:
- Aggregated name of the source pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | +| `src_namespace` | keyword | Namespace of the source endpoint. | +| `src_type` | keyword | Source endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload’s own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, the one with the longest-prefix match is chosen. Remaining ties are resolved alphabetically by the NetworkSet’s full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | +| `dest_name_aggr` | keyword | Contains one of the following values:
- Aggregated name of the destination pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | +| `dest_namespace` | keyword | Namespace of the destination endpoint. | +| `dest_type` | keyword | Destination endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload’s own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, CIDR matches outrank domain matches and longest-prefix wins between competing CIDR matches. Remaining ties are resolved alphabetically by the NetworkSet’s full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | +| `dest_service_name` | keyword | Name of the destination service. This may be empty if the request was not made against a service. | +| `dest_service_namespace` | keyword | Namespace of the destination service. This may be empty if the request was not made against a service. | +| `dest_service_port` | long | Destination service port. | +| `url` | keyword | URL that the request was made against. | +| `response_code` | keyword | Response code returned by the request. | +| `method` | keyword | HTTP method for the request. | +| `user_agent` | keyword | User agent of the request. | +| `type` | keyword | Type of request made. Possible values include `tcp`, `tls`, and `html/`. | diff --git a/calico-enterprise/observability/elastic/l7/datatypes.mdx b/calico-enterprise/observability/elastic/l7/datatypes.mdx index 478adeeb37..1e98127fa1 100644 --- a/calico-enterprise/observability/elastic/l7/datatypes.mdx +++ b/calico-enterprise/observability/elastic/l7/datatypes.mdx @@ -1,36 +1,47 @@ --- -description: Reference of key/value fields that Calico Enterprise sends to Elasticsearch for L7 logs, including durations, byte counts, and HTTP request metadata. +description: Reference of key/value fields that Calico Enterprise sends to Elasticsearch for L7 logs, with the collector that populates each field. --- # L7 log data types ## Big picture -$[prodname] sends the following data to Elasticsearch. +$[prodname] supports multiple L7 collectors that write into the same Elasticsearch index: -The following table details the key/value pairs in the JSON blob, including their [Elasticsearch datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html). +- **Envoy** — the legacy sidecar/proxy-based collector. See [Configure L7 logs](configure.mdx). +- **eBPF** — kernel TCP-layer probes capturing HTTP request/response and TLS handshake metadata. See [Configure L7 logs with eBPF](configure-bpf.mdx). +- **Waypoint** — collection through an Istio Ambient Mode Waypoint proxy. See [Configure L7 logs with Istio Waypoint](configure-istio-waypoint.mdx). -| Name | Datatype | Description | -| ------------------------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `host` | keyword | Name of the node that collected the L7 log entry. | -| `start_time` | date | Start time of log collection in Unix timestamp format. | -| `end_time` | date | End time of log collection in Unix timestamp format. | -| `bytes_in` | long | Number of incoming bytes since the last export. | -| `bytes_out` | long | Number of outgoing bytes since the last export. | -| `duration_mean` | long | Mean duration time of all the requests that match this combination of L7 data in nanoseconds. | -| `duration_max` | long | Max duration time of all the requests that match this combination of L7 data in nanoseconds. | -| `count` | long | Number of requests that match this combination of L7 data. | -| `src_name_aggr` | keyword | Contains one of the following values:
- Aggregated name of the source pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | -| `src_namespace` | keyword | Namespace of the source endpoint. | -| `src_type` | keyword | Source endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload’s own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, the one with the longest-prefix match is chosen. Remaining ties are resolved alphabetically by the NetworkSet’s full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | -| `dest_name_aggr` | keyword | Contains one of the following values:
- Aggregated name of the destination pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | -| `dest_namespace` | keyword | Namespace of the destination endpoint. | -| `dest_type` | keyword | Destination endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload’s own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, CIDR matches outrank domain matches and longest-prefix wins between competing CIDR matches. Remaining ties are resolved alphabetically by the NetworkSet’s full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | -| `dest_service_name` | keyword | Name of the destination service. This may be empty if the request was not made against a service. | -| `dest_service_namespace` | keyword | Namespace of the destination service. This may be empty if the request was not made against a service. | -| `dest_service_port` | long | Destination service port. | -| `url` | keyword | URL that the request was made against. | -| `response_code` | keyword | Response code returned by the request. | -| `method` | keyword | HTTP method for the request. | -| `user_agent` | keyword | User agent of the request. | -| `type` | keyword | Type of request made. Possible values include `tcp`, `tls`, and `html/`. | +The table below details the key/value pairs in the JSON blob, including their [Elasticsearch datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html) and which collector(s) populate each field. "All" means every collector populates the field. A qualifier like "eBPF (TLS only)" means the field is populated only for events of that protocol family. + +| Name | Datatype | Populated by | Description | +| ------------------------ | -------- | ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `host` | keyword | All | Name of the node that collected the L7 log entry. | +| `start_time` | date | All | Start time of log collection in Unix timestamp format. | +| `end_time` | date | All | End time of log collection in Unix timestamp format. | +| `bytes_in` | long | All | Number of incoming bytes since the last export. | +| `bytes_out` | long | All | Number of outgoing bytes since the last export. | +| `duration_mean` | long | All | Mean duration time of all the requests that match this combination of L7 data in nanoseconds. | +| `duration_max` | long | All | Max duration time of all the requests that match this combination of L7 data in nanoseconds. | +| `count` | long | All | Number of requests that match this combination of L7 data. | +| `src_name_aggr` | keyword | All | Contains one of the following values:
- Aggregated name of the source pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | +| `src_namespace` | keyword | All | Namespace of the source endpoint. | +| `src_type` | keyword | All | Source endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload's own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, the one with the longest-prefix match is chosen. Remaining ties are resolved alphabetically by the NetworkSet's full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | +| `dest_name_aggr` | keyword | All | Contains one of the following values:
- Aggregated name of the destination pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | +| `dest_namespace` | keyword | All | Namespace of the destination endpoint. | +| `dest_type` | keyword | All | Destination endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload's own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, CIDR matches outrank domain matches and longest-prefix wins between competing CIDR matches. Remaining ties are resolved alphabetically by the NetworkSet's full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | +| `dest_service_name` | keyword | All | Name of the destination service. This may be empty if the request was not made against a service. | +| `dest_service_namespace` | keyword | All | Namespace of the destination service. This may be empty if the request was not made against a service. | +| `dest_service_port` | long | All | Destination service port. | +| `url` | keyword | All (HTTP only) | URL that the request was made against. | +| `response_code` | keyword | All (HTTP only) | Response code returned by the request. | +| `method` | keyword | All (HTTP only) | HTTP method for the request. | +| `user_agent` | keyword | Envoy, Waypoint | User agent of the request. | +| `type` | keyword | Envoy, Waypoint | Type of request made. Possible values include `tcp`, `tls`, and `html/`. | +| `protocol` | keyword | eBPF, Waypoint | Wire-protocol family of the captured event. Possible values: `http`, `tls`. | +| `protocol_version` | keyword | eBPF, Waypoint | Wire-protocol version. For HTTP events, possible values include `1.0` and `1.1`. | +| `collector_type` | keyword | eBPF, Waypoint | Category of the collector that produced the event. Possible values include `ebpf` and `waypoint`. | +| `collector_name` | keyword | eBPF, Waypoint | Name of the specific collector that produced the event. For the kernel TCP-layer eBPF collector, the value is `ebpf-tcp`. | +| `tls_sni` | keyword | eBPF (TLS only) | TLS Server Name Indication sent by the client in the ClientHello. | +| `tls_version` | keyword | eBPF (TLS only) | Negotiated TLS protocol version (for example, `TLS 1.2`, `TLS 1.3`). | +| `tls_cipher_suite` | keyword | eBPF (TLS only) | Negotiated TLS cipher suite (for example, `TLS_AES_256_GCM_SHA384`). | diff --git a/sidebars-calico-enterprise.js b/sidebars-calico-enterprise.js index 5c9e7a3f73..b5835dbc7a 100644 --- a/sidebars-calico-enterprise.js +++ b/sidebars-calico-enterprise.js @@ -402,7 +402,14 @@ module.exports = { type: 'category', label: 'L7 logs', link: { type: 'doc', id: 'observability/elastic/l7/index' }, - items: ['observability/elastic/l7/configure', 'observability/elastic/l7/datatypes'], + items: [ + { type: 'doc', id: 'observability/elastic/l7/configure-bpf', label: '+ Configure with eBPF' }, + { type: 'doc', id: 'observability/elastic/l7/configure-istio-waypoint', label: '+ Configure with Istio Waypoint' }, + { type: 'doc', id: 'observability/elastic/l7/datatypes', label: '+ Data types' }, + { type: 'doc', id: 'observability/elastic/l7/aggregation', label: '+ Aggregation' }, + 'observability/elastic/l7/configure', + { type: 'doc', id: 'observability/elastic/l7/datatypes-legacy', label: 'L7 log data types' }, + ], }, 'observability/elastic/troubleshoot', ],