WIP: docs for eBPF L7 observability

calico-enterprise/_includes/components/L7AggregationSandbox/index.js

@@ -0,0 +1,329 @@
+import React, { useState, useMemo, useEffect, useRef } from 'react';
+
+const SAMPLE = [
+  { src: 'front',  dest: 'cart',   svc: 'cart-svc',   method: 'GET',  url: '/cart',   code: '200', ua: 'client', protocol: 'http', sni: null },
+  { src: 'front',  dest: 'cart',   svc: 'cart-svc',   method: 'GET',  url: '/cart',   code: '404', ua: 'client', protocol: 'http', sni: null },
+  { src: 'front',  dest: 'cart',   svc: 'cart-svc',   method: 'GET',  url: '/items',  code: '200', ua: 'client', protocol: 'http', sni: null },
+  { src: 'front',  dest: 'cart',   svc: 'cart-svc',   method: 'POST', url: '/cart',   code: '200', ua: 'client', protocol: 'http', sni: null },
+  { src: 'orders', dest: 'pay',    svc: 'pay-svc',    method: 'POST', url: '/charge', code: '200', ua: 'sdk',    protocol: 'http', sni: null },
+  { src: 'edge',   dest: 'pay',    svc: 'pay-svc',    method: 'POST', url: '/charge', code: '200', ua: 'sdk',    protocol: 'http', sni: null },
+  { src: 'orders', dest: 'canary', svc: 'pay-svc',    method: 'POST', url: '/charge', code: '200', ua: 'sdk',    protocol: 'http', sni: null },
+  { src: 'orders', dest: 'pay',    svc: 'pay-v2-svc', method: 'POST', url: '/charge', code: '200', ua: 'sdk',    protocol: 'http', sni: null },
+  { src: 'prom',   dest: 'kibana', svc: 'kibana-svc', method: null,   url: null,      code: null,  ua: null,     protocol: 'tls',  sni: 'kibana.svc' },
+  { src: 'prom',   dest: 'kibana', svc: 'kibana-svc', method: null,   url: null,      code: null,  ua: null,     protocol: 'tls',  sni: 'es.svc' },
+];
+
+const AGGREGATORS = [
+  { id: 'source',  label: 'Source info',      fields: ['src'],    field: 'L7LogsFileAggregationSourceInfo' },
+  { id: 'dest',    label: 'Destination info', fields: ['dest'],   field: 'L7LogsFileAggregationDestinationInfo' },
+  { id: 'service', label: 'Service info',     fields: ['svc'],    field: 'L7LogsFileAggregationServiceInfo' },
+  { id: 'method',  label: 'HTTP method',      fields: ['method'], field: 'L7LogsFileAggregationHTTPMethod' },
+  { id: 'url',     label: 'URL',              fields: ['url'],    field: 'L7LogsFileAggregationTrimURL' },
+  { id: 'code',    label: 'Response code',    fields: ['code'],   field: 'L7LogsFileAggregationResponseCode' },
+  { id: 'header',  label: 'HTTP header info', fields: ['ua'],     field: 'L7LogsFileAggregationHTTPHeaderInfo' },
+  { id: 'tlssni',  label: 'TLS SNI',          fields: ['sni'],    field: 'L7LogsFileAggregationTLSSNI' },
+];
+
+const COLUMNS = [
+  { key: 'protocol', label: 'protocol',          owner: null },
+  { key: 'src',      label: 'src_name_aggr',     owner: 'source' },
+  { key: 'dest',     label: 'dest_name_aggr',    owner: 'dest' },
+  { key: 'svc',      label: 'dest_service_name', owner: 'service' },
+  { key: 'method',   label: 'method',            owner: 'method' },
+  { key: 'url',      label: 'url',               owner: 'url' },
+  { key: 'code',     label: 'response_code',     owner: 'code' },
+  { key: 'ua',       label: 'user_agent',        owner: 'header' },
+  { key: 'sni',      label: 'tls_sni',           owner: 'tlssni' },
+];
+
+const HIGHLIGHT_MS = 1500;
+
+export default function L7AggregationSandbox() {
+  const [checked, setChecked] = useState(
+    Object.fromEntries(AGGREGATORS.map((a) => [a.id, true])),
+  );
+
+  const displayedRows = useMemo(() => {
+    const bucketKeys = SAMPLE.map((row) => {
+      const parts = [`protocol:${row.protocol}`];
+      for (const a of AGGREGATORS) {
+        if (!checked[a.id]) continue;
+        for (const f of a.fields) parts.push(`${f}:${row[f] ?? ''}`);
+      }
+      return parts.join('|');
+    });
+
+    const bucketCounts = new Map();
+    const bucketOrder = new Map();
+    let nextBucketIdx = 0;
+    for (const k of bucketKeys) {
+      bucketCounts.set(k, (bucketCounts.get(k) || 0) + 1);
+      if (!bucketOrder.has(k)) {
+        bucketOrder.set(k, nextBucketIdx++);
+      }
+    }
+    const primaryRowForBucket = new Map();
+
+    const rows = SAMPLE.map((row, originalIdx) => {
+      const k = bucketKeys[originalIdx];
+      if (!primaryRowForBucket.has(k)) primaryRowForBucket.set(k, originalIdx);
+      const primaryOrigIdx = primaryRowForBucket.get(k);
+      const isPrimary = primaryOrigIdx === originalIdx;
+      const bucketIdx = bucketOrder.get(k);
+      const bucketSize = bucketCounts.get(k);
+      const out = {
+        originalIdx,
+        bucketIdx,
+        primaryOrigIdx,
+        bucketSize,
+        count: isPrimary ? bucketSize : '',
+        __primary: isPrimary,
+      };
+      for (const col of COLUMNS) {
+        out[col.key] =
+          col.owner && !checked[col.owner] ? '—' : row[col.key] ?? '—';
+      }
+      return out;
+    });
+
+    rows.sort((a, b) => {
+      if (a.bucketIdx !== b.bucketIdx) return a.bucketIdx - b.bucketIdx;
+      return a.originalIdx - b.originalIdx;
+    });
+
+    return rows;
+  }, [checked]);
+
+  const prevRef = useRef(displayedRows);
+  const [highlights, setHighlights] = useState(() => new Map());
+  const [expanded, setExpanded] = useState(() => new Set());
+
+  const toggleExpand = (originalIdx) => {
+    setExpanded((prev) => {
+      const next = new Set(prev);
+      if (next.has(originalIdx)) next.delete(originalIdx);
+      else next.add(originalIdx);
+      return next;
+    });
+  };
+
+  const visibleRows = displayedRows.filter(
+    (row) => row.__primary || expanded.has(row.primaryOrigIdx),
+  );
+
+  useEffect(() => {
+    const newHL = new Map();
+    const prevByOrigIdx = new Map();
+    for (const r of prevRef.current) {
+      prevByOrigIdx.set(r.originalIdx, r);
+    }
+    const HIGHLIGHTABLE = ['count', ...COLUMNS.map((c) => c.key)];
+    const numOrZero = (v) => (typeof v === 'number' ? v : 0);
+    for (const row of displayedRows) {
+      const prev = prevByOrigIdx.get(row.originalIdx) || {};
+      for (const k of HIGHLIGHTABLE) {
+        if (row[k] === prev[k]) continue;
+        let direction;
+        if (k === 'count') {
+          direction = numOrZero(row[k]) > numOrZero(prev[k]) ? 'add' : 'remove';
+        } else {
+          direction = row[k] === '—' ? 'remove' : 'add';
+        }
+        newHL.set(`${row.originalIdx}:${k}`, direction);
+      }
+    }
+    prevRef.current = displayedRows;
+    setHighlights(newHL);
+    if (newHL.size === 0) return;
+    const t = setTimeout(() => setHighlights(new Map()), HIGHLIGHT_MS);
+    return () => clearTimeout(t);
+  }, [displayedRows]);
+
+  const toggle = (id) => setChecked((c) => ({ ...c, [id]: !c[id] }));
+
+  const ROW_HEIGHT = '3.2em';
+  const baseCellStyle = {
+    padding: '4px 8px',
+    fontFamily: 'var(--ifm-font-family-monospace)',
+    fontSize: '0.85em',
+    overflowWrap: 'anywhere',
+    verticalAlign: 'top',
+    transition: 'background-color 1s ease',
+    height: ROW_HEIGHT,
+    boxSizing: 'border-box',
+    backgroundColor: 'transparent',
+  };
+  const cellStyleFor = (rowIdx, key) => {
+    const direction = highlights.get(`${rowIdx}:${key}`);
+    if (direction === 'add') {
+      return {
+        ...baseCellStyle,
+        backgroundColor: 'var(--ifm-color-success-contrast-background)',
+      };
+    }
+    if (direction === 'remove') {
+      return {
+        ...baseCellStyle,
+        backgroundColor: 'var(--ifm-color-danger-contrast-background)',
+      };
+    }
+    return baseCellStyle;
+  };
+  const headerStyle = {
+    ...baseCellStyle,
+    whiteSpace: 'normal',
+    wordBreak: 'break-word',
+    transition: 'none',
+  };
+
+  return (
+    <div>
+      <div
+        style={{
+          display: 'flex',
+          flexWrap: 'wrap',
+          gap: '8px 16px',
+          margin: '12px 0',
+          padding: '12px',
+          border: '1px solid var(--ifm-color-emphasis-300)',
+          borderRadius: '4px',
+        }}
+      >
+        {AGGREGATORS.map((a) => (
+          <label
+            key={a.id}
+            style={{
+              display: 'inline-flex',
+              alignItems: 'center',
+              gap: 4,
+              cursor: 'pointer',
+            }}
+          >
+            <input
+              type="checkbox"
+              checked={checked[a.id]}
+              onChange={() => toggle(a.id)}
+            />
+            <span>{a.label}</span>
+          </label>
+        ))}
+      </div>
+      <div
+        style={{
+          display: 'flex',
+          gap: '16px',
+          margin: '8px 0',
+          fontSize: '0.85em',
+          opacity: 0.8,
+        }}
+      >
+        <span style={{ display: 'inline-flex', alignItems: 'center', gap: 4 }}>
+          <span
+            style={{
+              display: 'inline-block',
+              width: '0.9em',
+              height: '0.9em',
+              backgroundColor: 'var(--ifm-color-success-contrast-background)',
+              border: '1px solid var(--ifm-color-emphasis-300)',
+              borderRadius: '2px',
+            }}
+          />
+          added (value gained or count up)
+        </span>
+        <span style={{ display: 'inline-flex', alignItems: 'center', gap: 4 }}>
+          <span
+            style={{
+              display: 'inline-block',
+              width: '0.9em',
+              height: '0.9em',
+              backgroundColor: 'var(--ifm-color-danger-contrast-background)',
+              border: '1px solid var(--ifm-color-emphasis-300)',
+              borderRadius: '2px',
+            }}
+          />
+          removed (value lost or count down)
+        </span>
+      </div>
+      <div style={{ overflowX: 'auto' }}>
+        <table style={{ tableLayout: 'fixed', width: '100%' }}>
+          <colgroup>
+            <col style={{ width: '10%' }} />
+            {COLUMNS.map((c) => (
+              <col key={c.key} style={{ width: `${90 / COLUMNS.length}%` }} />
+            ))}
+          </colgroup>
+          <thead>
+            <tr>
+              <th style={headerStyle}>count</th>
+              {COLUMNS.map((c) => (
+                <th key={c.key} style={headerStyle}>{c.label}</th>
+              ))}
+            </tr>
+          </thead>
+          <tbody>
+            {visibleRows.map((row, displayIdx) => {
+              const prevRow = visibleRows[displayIdx - 1];
+              const isBucketStart =
+                !prevRow || prevRow.bucketIdx !== row.bucketIdx;
+              const borderTop = isBucketStart && displayIdx > 0
+                ? '2px solid var(--ifm-color-emphasis-300)'
+                : undefined;
+              const styleForCell = (key) => {
+                const base = cellStyleFor(row.originalIdx, key);
+                return borderTop ? { ...base, borderTop } : base;
+              };
+              const rowOpacity = row.__primary ? 1 : 0.7;
+              const isExpandable = row.__primary && row.bucketSize > 1;
+              const isOpen = expanded.has(row.originalIdx);
+              return (
+                <tr
+                  key={row.originalIdx}
+                  style={{
+                    height: ROW_HEIGHT,
+                    opacity: rowOpacity,
+                    transition: 'opacity 0.4s ease',
+                    backgroundColor: 'transparent',
+                  }}
+                >
+                  <td style={{ ...styleForCell('count'), fontWeight: row.__primary ? 600 : 400 }}>
+                    {isExpandable ? (
+                      <button
+                        type="button"
+                        onClick={() => toggleExpand(row.originalIdx)}
+                        style={{
+                          display: 'inline-flex',
+                          alignItems: 'center',
+                          justifyContent: 'center',
+                          width: '1.5em',
+                          height: '1.5em',
+                          marginRight: 6,
+                          padding: 0,
+                          background: 'var(--ifm-color-emphasis-100)',
+                          border: '1px solid var(--ifm-color-emphasis-400)',
+                          borderRadius: '3px',
+                          cursor: 'pointer',
+                          color: 'var(--ifm-color-emphasis-800)',
+                          fontFamily: 'inherit',
+                          fontSize: '1em',
+                          fontWeight: 700,
+                          lineHeight: 1,
+                        }}
+                        aria-label={isOpen ? 'Collapse group' : 'Expand group'}
+                      >
+                        {isOpen ? '−' : '+'}
+                      </button>
+                    ) : null}
+                    {row.count}
+                  </td>
+                  {COLUMNS.map((c) => (
+                    <td key={c.key} style={styleForCell(c.key)}>{row[c.key]}</td>
+                  ))}
+                </tr>
+              );
+            })}
+          </tbody>
+        </table>
+      </div>
+    </div>
+  );
+}

calico-enterprise/observability/elastic/l7/aggregation.mdx

@@ -0,0 +1,41 @@
+---
+description: Control how Calico Enterprise aggregates L7 log entries by selecting which fields participate in the aggregation key.
+---
+
+import L7AggregationSandbox from '../../../_includes/components/L7AggregationSandbox';
+
+# L7 log aggregation
+
+## Big picture
+
+$[prodname] aggregates L7 events before writing them to disk. Each row in the resulting log represents a unique combination of metadata fields and carries a `count` of how many requests matched that combination during the aggregation interval. Which fields participate in that combination — the aggregation key — is controlled by a set of fields on [Felix Configuration](../../../reference/component-resources/node/felix/configuration.mdx#l7-logs).
+
+Including a field makes the logs more granular (more rows, fewer requests per row). Excluding a field collapses requests that differ only in that field into a single row and increments its `count` for each merged request.
+
+## Aggregation playground
+
+The table below shows a small set of sample L7 events. Each visible row is what would be written to the L7 log, with `count` equal to the number of requests merged into it. Toggle the checkboxes to include or exclude each field from the aggregation key.
+
+<L7AggregationSandbox />
+
+## Aggregation fields
+
+The following `FelixConfiguration` fields control the L7 log aggregation key. For accepted values and defaults, follow the link to the [Felix Configuration reference](../../../reference/component-resources/node/felix/configuration.mdx#l7-logs).
+
+| Field | Effect when included | Effect when excluded |
+| --- | --- | --- |
+| [L7LogsFileAggregationSourceInfo](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationSourceInfo) | Source aggregated name, namespace, and type appear on each row. | Requests are merged across sources. |
+| [L7LogsFileAggregationDestinationInfo](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationDestinationInfo) | Destination aggregated name, namespace, and type appear on each row. | Requests are merged across destinations. |
+| [L7LogsFileAggregationServiceInfo](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationServiceInfo) | Destination service name, namespace, and port appear on each row. | Requests are merged across services. |
+| [L7LogsFileAggregationHTTPMethod](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationHTTPMethod) | HTTP method appears on each row. | Requests are merged across methods. |
+| [L7LogsFileAggregationTrimURL](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationTrimURL) | URL (or a trimmed prefix) appears on each row. Intermediate values trim the query string, then the path, before excluding the URL entirely. | Requests are merged across URLs. |
+| [L7LogsFileAggregationResponseCode](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationResponseCode) | Response code appears on each row. | Requests are merged across response codes. |
+| [L7LogsFileAggregationHTTPHeaderInfo](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationHTTPHeaderInfo) | User agent and request type appear on each row. | Requests are merged across user agents and types. |
+| [L7LogsFileAggregationTLSSNI](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationTLSSNI) | TLS Server Name Indication appears on each TLS row. | TLS flows to the same destination are merged regardless of SNI. |
+
+Two additional fields control how the URL is truncated before it enters the aggregation key, rather than whether the URL participates at all:
+
+| Field | Description |
+| --- | --- |
+| [L7LogsFileAggregationNumURLPath](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationNumURLPath) | Maximum number of URL path components retained. Default `5`. A negative value disables this truncation. |
+| [L7LogsFileAggregationURLCharLimit](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationURLCharLimit) | Maximum URL length in characters; longer URLs are sliced. Default `250`. |