diff --git a/.gitignore b/.gitignore index 6d8fa96..15b38bc 100644 --- a/.gitignore +++ b/.gitignore @@ -18,10 +18,12 @@ node_modules/ *.log npm-debug.log* -# Internal scripts and metadata -scripts/ +# Internal scratch metadata workflow-metadata.json +# Build output (generated by scripts/build-catalog.mjs; uploaded to the CDN) +dist/ + # Environment and secrets .env .env.* diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 505421c..1891306 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,316 +1,231 @@ -# Contributing to Elastic Workflows Library +# Contributing to the Elastic Workflow Template Library -Thank you for your interest in contributing to the Elastic Workflows Library! This document provides guidelines for contributing workflows and improvements. +Thanks for your interest in contributing. This document is the authoring guide for the **Workflow Template Library** that ships in Kibana (Tech Preview from 9.5). Read it before opening your first PR. --- -## Table of Contents - -- [Ways to Contribute](#ways-to-contribute) -- [Workflow Contribution Guidelines](#workflow-contribution-guidelines) -- [Submission Process](#submission-process) -- [Code of Conduct](#code-of-conduct) +## Table of contents + +- [Ways to contribute](#ways-to-contribute) +- [Authoring a template](#authoring-a-template) + - [File layout](#file-layout) + - [The `template-metadata` block](#the-template-metadata-block) + - [Categories vocabulary](#categories-vocabulary) + - [Install-time inputs (`install.form` + `__install__.*`)](#install-time-inputs-installform--__install__) + - [Step types and connectors](#step-types-and-connectors) + - [Style and idiomatic patterns](#style-and-idiomatic-patterns) +- [Validating locally](#validating-locally) +- [Versioning](#versioning) +- [Pull request flow](#pull-request-flow) +- [Code of conduct](#code-of-conduct) --- -## Ways to Contribute - -### 1. Submit New Workflows - -Share workflows you've developed for: -- Threat detection and alerting -- Incident response automation -- Data enrichment with threat intelligence -- Threat hunting queries -- Integration with external tools -- Scheduled maintenance tasks - -### 2. Improve Existing Workflows - -- Fix bugs or issues -- Improve documentation and comments -- Optimize performance -- Add error handling -- Extend functionality - -### 3. Improve Documentation +## Ways to contribute -- Fix typos or unclear explanations -- Add examples -- Improve schema documentation -- Translate to other languages - -### 4. Report Issues - -- Report bugs in workflows -- Suggest improvements -- Request new workflow types +1. **Add a new template.** Submit a new YAML under `library/workflows//.yaml` with a valid `template-metadata` block. +2. **Improve an existing template.** Tighten a description, fix a bug, swap a generic `http` step for a dedicated vendor step type, add helpful install-form fields. +3. **Extend the categories vocabulary.** When a new template genuinely needs a category not in `library/categories.yaml`, add the entry in the same PR. +4. **Improve documentation.** Fix unclear wording, add examples, clarify the authoring rules. +5. **Report issues.** File a GitHub issue for bugs, suggestions, or missing capabilities. --- -## Workflow Contribution Guidelines +## Authoring a template -### File Structure +### File layout -Place your workflow in the appropriate category: +Every template lives at: ``` -workflows/ -├── detection/ # Threat detection, alerting -├── response/ # Incident response, case management -├── enrichment/ # Threat intel, data enrichment -├── hunting/ # Threat hunting queries -├── notification/ # Alerts, messaging -├── automation/ # Scheduled tasks, utilities -└── uncategorized/ # Miscellaneous +library/workflows//.yaml ``` -### Naming Conventions - -- Use lowercase with hyphens: `ip-reputation-check.yaml` -- Be descriptive: `virustotal-hash-lookup.yaml` not `vt-lookup.yaml` -- Include the primary action: `slack-alert-notification.yaml` +- The `` directory name and the YAML file name **must match the `slug` value** inside the file's `template-metadata` block. The catalog generator enforces this. +- Slug format: kebab-case, lowercase ASCII alphanumeric + hyphens. Should be descriptive and unique across the library. +- One template per directory. Future multi-version coexistence will live as `/-v.yaml` siblings, but the starter set is all v1. -### Required Documentation +### The `template-metadata` block -Every workflow must include: - -#### 1. Header Comment Block +Top of every template file. The body that follows is regular workflow YAML grammar (`consts:`, `inputs:` / `triggers:`, `steps:`, …). ```yaml -# ============================================================================= -# Workflow: [Descriptive Name] -# -# [2-4 sentence description of what this workflow does, when to use it, -# and what it integrates with] -# -# Author: [Your Name] -# Date: [YYYY-MM-DD] -# Tags: [tag1, tag2, tag3] -# ============================================================================= +template-metadata: + slug: ip-reputation-check # MUST match parent dir name + version: "1.0.0" # semver; bump on every content change + availability: ">=9.5.0" # semver range over Kibana versions + name: "IP Reputation Check (AbuseIPDB)" + description: >- + Assess the reputation of an IP address using AbuseIPDB and enrich + with geolocation data. Produces a low / medium / high risk verdict. + solutions: [security] # optional. absent or empty = cross-solution (every solution context) + categories: [enrichment, threat-intel] # closed-vocab; entries MUST exist in library/categories.yaml + icon: abuseipdb # optional; references a known icon ID + install: # only required when the body uses __install__. + form: + - name: abuseipdb-connector + label: "AbuseIPDB connector" + description: "The AbuseIPDB connector used to query IP reputation." + inputType: connector + connectorType: .abuseipdb + required: true ``` -#### 2. Section Comments +**Required fields:** `slug`, `version`, `availability`, `name`, `description`, `categories`. +**Optional fields:** `solutions`, `icon`, `install`. -```yaml -# --------------------------------------------------------------------------- -# CONSTANTS -# [Brief description of what needs to be configured] -# --------------------------------------------------------------------------- -consts: - api_key: "YOUR-API-KEY" # Explain where to get this -``` +Notes on the optional fields: -#### 3. Step Comments +- **`solutions`** — when present, an array of solution ids (e.g. `[security]`, `[security, observability]`). Absent or empty means the template is **cross-solution** and appears in every solution context. +- **`icon`** — references an icon known to the Kibana UI (e.g. `abuseipdb`, `slack`, `virustotal`). Omit if there's no obvious match yet. +- **`install`** — required if and only if the workflow body references any `__install__.` placeholder. See [Install-time inputs](#install-time-inputs-installform--__install__) below. -```yaml -steps: - # Step 1: [What this step does] - # Purpose: [Why this step is needed] - - name: descriptive_step_name - type: action.type -``` +### Categories vocabulary -### Code Quality +`categories: [...]` is a **closed vocabulary**. Every value used in any template's `categories` array must exist as an `id` in [`library/categories.yaml`](./library/categories.yaml). The catalog generator rejects any template referencing an unknown id. -#### Use Meaningful Names +If your template genuinely needs a category that is not in the vocab, **add the entry to `library/categories.yaml` in the same PR** — never invent values used only in a template. Reviewers will either accept the new entry or point you at an existing one. -```yaml -# ❌ Bad -- name: step1 - type: http - with: - url: "{{ consts.u }}" - -# ✅ Good -- name: lookup_ip_reputation - type: http - with: - url: "{{ consts.virustotal_api_url }}/ip-address/{{ inputs.ip }}" -``` +### Install-time inputs (`install.form` + `__install__.*`) -#### Include Error Handling +When the operator installs a template into their Kibana, the catalog UI renders an install form derived from `template-metadata.install.form`. Whatever values the user submits get substituted into the workflow body wherever it references `__install__.`. -```yaml -- name: external_api_call - type: http - with: - url: "{{ consts.api_url }}" - on-failure: - retry: - max-attempts: 3 - delay: 5s - continue: true # Or false if critical -``` +Two rules to internalize: -#### Use Constants for Configuration +1. **`install.form` is the single source of truth.** Every `__install__.` reference in the body MUST have a matching entry in `install.form`. The installer does not auto-derive form fields from `consts:` or anywhere else; an undeclared reference fails the install. +2. **Form field names are kebab-case by convention** (e.g. `abuseipdb-connector`, `max-age-in-days`). They are internal template identifiers and are substituted away during rendering — end users never see them in the final workflow YAML. -```yaml -# ✅ Good - Easy to customize -consts: - api_endpoint: "https://api.example.com/v2" - timeout_seconds: 30 - max_results: 100 - -steps: - - name: query_api - with: - url: "{{ consts.api_endpoint }}/search" - timeout: "{{ consts.timeout_seconds }}s" -``` +What belongs in the install form (vs `consts:`): -### Security Requirements +- **Promote to `install.form`** anything the operator must configure for the template to work: connector ids (always), Slack channels, recipient emails, tunable thresholds you want to expose in the install UX, environment-specific URLs. +- **Keep in `consts:`** stable, non-secret config that does not vary per installation: vendor base URLs (when the dedicated connector doesn't own them), hard-coded defaults the install UX does not need to expose. -#### 1. Never Commit Real Credentials +Templates that don't need any install-time inputs **omit the `install:` block entirely**. -```yaml -# ❌ Bad -consts: - api_key: "sk-abc123xyz789" # Real API key! +#### `inputType` reference -# ✅ Good -consts: - api_key: "YOUR-API-KEY-HERE" # Placeholder -``` +| `inputType` | Purpose | Required extras | +|---|---|---| +| `text` | Free-form short string | — | +| `textarea` | Multi-line text | — | +| `number` | Numeric input | — | +| `boolean` | Toggle | — | +| `select` | Single choice from a fixed list | `options: [{ value, label }, ...]` | +| `connector` | Picks an existing Kibana stack connector | `connectorType: .` | -#### 2. Document Required Permissions +Every field can carry `label`, `description`, `required` (default `false`), and `default`. -```yaml -# ============================================================================= -# Workflow: Case Creator -# -# Required Permissions: -# - Kibana: Cases > All -# - Elasticsearch: Read access to .alerts-* indices -# ============================================================================= -``` +#### Connector type rule -#### 3. Validate Inputs +`install.form[].connectorType` must equal `.` + the prefix of the step type that uses it. For example: -```yaml -inputs: - - name: ip_address - type: string - description: "IPv4 or IPv6 address (validated format)" - required: true -``` +- `abuseipdb.checkIp` → `connectorType: .abuseipdb` +- `virustotal.scanFileHash` → `connectorType: .virustotal` +- `slack2.createConversation` → `connectorType: .slack2` +- `brave-search.webSearch` → `connectorType: .brave-search` -### Testing Requirements +Never use `.webhook` as a connector type; always pick the dedicated `.` connector. -Before submitting, verify: +### Step types and connectors -1. **YAML Syntax**: Passes `yamllint` or similar -2. **Schema Validation**: Imports into Kibana without errors -3. **Functional Test**: Executes successfully with test data -4. **Error Handling**: Behaves correctly when external services fail +The workflow engine's step-type registry is the source of truth — refer to the JSON schema published by `@kbn/workflows` for the canonical list. ---- +Two rules: -## Submission Process +1. **Prefer the dedicated vendor step type.** If a vendor has a dedicated step (e.g. `abuseipdb.checkIp`, `virustotal.scanFileHash`, `slack2.createConversation`, `brave-search.webSearch`), use it. The legacy generic `http` step is an escape hatch and should only appear when no dedicated step exists. +2. **Never invent a step type or a connector type.** If you think one is missing, file an issue rather than working around it locally. -### Step 1: Fork the Repository +### Style and idiomatic patterns -```bash -# Fork on GitHub, then clone -git clone https://github.com/YOUR_USERNAME/elastic-workflows.git -cd elastic-workflows -``` +- **2-space YAML indentation.** +- **No `id:`, no `metadata:` (singular), no `since`/`discontinued`/`replacement` fields.** Those are obsolete shapes from earlier drafts. +- **Drop the legacy banner headers.** No `# =================== Workflow: X` block at the top, no `# CONSTANTS / # INPUTS / # TRIGGERS / # STEPS` tutorial blocks. The `template-metadata` block is the header. +- **Keep per-step comments, trimmed.** One short paragraph per step explaining intent. Avoid restating what the YAML already says. +- **Snake_case for workflow-body identifiers** (input names, step names): `ip_address`, `check_abuseipdb`, `format_results`. Kebab-case is reserved for `install.form` field names. +- **Prefer the dedicated `data.*` step types over abusing `console` for value transformation.** Use `data.parseJson`, `data.set`, etc. when you want to compute or restructure data. + +--- -### Step 2: Create a Branch +## Validating locally + +The repo ships a small Node script that walks every template, validates its `template-metadata` block, cross-checks `__install__.` against `install.form`, verifies every `categories[]` entry against the vocab, and produces the per-Kibana-version catalogs the CDN serves. ```bash -git checkout -b add-workflow/my-new-workflow -# or -git checkout -b fix/workflow-name-issue +npm install +npm run build:catalog ``` -### Step 3: Add Your Workflow - -1. Create the YAML file in the appropriate category folder -2. Include all required documentation (header, section comments, step comments) -3. Test the workflow in your Kibana environment +Run it before submitting a PR. A non-zero exit means the catalog publish would fail; the error messages point at the offending file and field. -### Step 4: Update Category README (Optional) +### Env-var overrides -If adding a significant workflow, add it to the category's README.md: +The script resolves the live `main` Kibana semver and the supported named minors at run time. Two env vars let you skip those network calls for fast local iteration: -```markdown -| [My Workflow](./my-workflow.yaml) | Brief description | Author | -``` +| Env var | Effect | +|---|---| +| `KIBANA_MAIN_VERSION=9.6.0` | Skip the fetch of `elastic/kibana@main`'s `package.json`. | +| `KIBANA_NAMED_MINORS=""` (or `"9.5,9.6"`) | Skip the GitHub branches API. Empty string = treat as zero named minors. | +| `GITHUB_TOKEN` | When set, authenticates the branches API call (5,000/h instead of 60/h). CI provides this automatically; locally, `export GITHUB_TOKEN=$(gh auth token)` works. | -### Step 5: Commit and Push +The fastest fully-offline iteration: ```bash -git add workflows/category/my-workflow.yaml -git commit -m "Add: My New Workflow for XYZ use case" -git push origin add-workflow/my-new-workflow +KIBANA_MAIN_VERSION=9.6.0 KIBANA_NAMED_MINORS="" npm run build:catalog ``` -### Step 6: Submit Pull Request - -1. Go to the original repository on GitHub -2. Click "New Pull Request" -3. Select your branch -4. Fill in the PR template: - -```markdown -## Description -[What does this workflow do?] +### What "valid" means -## Category -[detection/response/enrichment/etc.] +The script enforces: -## Testing -- [ ] Tested in Kibana [version] -- [ ] YAML validates without errors -- [ ] Workflow executes successfully +- File parses with `js-yaml`. +- `template-metadata` is present with all required fields. +- `slug` matches the parent directory name. +- `version` is valid semver; `availability` is a valid semver range. +- Every `categories[]` entry exists in `library/categories.yaml`. +- Every `__install__.` reference in the body has a matching `install.form` entry (and no orphan form fields). -## Checklist -- [ ] Header comment block included -- [ ] Section comments included -- [ ] Step comments included -- [ ] No real credentials/secrets -- [ ] Error handling configured -``` +It does **not** yet validate the workflow body against the engine's step-type schema — that runs in a sibling CI job once the validator package is published. Until then, the safest check is to install your template into a local Kibana and run it. -### Review Process +--- -1. Maintainers review the PR -2. Automated checks run (YAML validation, linting) -3. Feedback provided if changes needed -4. Once approved, PR is merged +## Versioning ---- +`template-metadata.version` is a semver. Bump it on every meaningful content change to a template body: -## Code of Conduct +- **Patch** (`1.0.0 → 1.0.1`) — typo fix, comment tweak, no behaviour change. +- **Minor** (`1.0.0 → 1.1.0`) — additive behaviour, new optional install field, additional step that doesn't affect existing callers. +- **Major** (`1.0.0 → 2.0.0`) — breaking change to inputs, install form, or the workflow's observable behaviour. -### Our Standards +`template-metadata.availability` is a semver range over Kibana versions. For now, every template carries `>=9.5.0`. When future Kibana versions retire a step type or connector convention, restrict the range accordingly (`">=9.5.0 <9.8.0"`) and ship a successor template under the same slug with a bumped major. -- Be respectful and inclusive -- Provide constructive feedback -- Focus on the contribution, not the contributor -- Accept criticism gracefully +Multi-version coexistence (the `/-v2.yaml` sibling layout) lands in Kibana 9.6; for the 9.5 starter set, every template is v1. -### Unacceptable Behavior +--- -- Harassment or discrimination -- Trolling or inflammatory comments -- Publishing others' private information -- Submitting malicious code +## Pull request flow -### Reporting +1. **Fork** the repo and clone your fork. +2. **Branch** from `main`: `git checkout -b add/` or `fix/-`. +3. **Author** the template under `library/workflows//.yaml`. Follow the rules above. +4. **Run the validator**: `npm run build:catalog` (with overrides as needed). Resolve any errors. +5. **Commit** with a clear message: `Add template` or `Fix : `. +6. **Open a PR** with: + - The slug and one-line description. + - Any migration decisions worth flagging (e.g. "promoted `X` from `consts:` to install form", "swapped raw `http` for dedicated `vendor.action` step"). + - Validation output (paste the last few lines of `npm run build:catalog`). + - A short test plan (e.g. "installed in local 9.5, ran with `ip_address=8.8.8.8`, output report rendered as expected"). -Report issues to the maintainers via GitHub Issues or email. +A maintainer will review and either request changes or merge. Once merged, the next push to `main` republishes the catalog and the template becomes installable from the Kibana UI on every active Kibana version whose semver satisfies your `availability:` range. --- -## Questions? +## Code of conduct -- Open a GitHub Issue for questions -- Check existing workflows for examples -- Review the [schema documentation](./docs/schema.md) +- Be respectful and constructive in reviews. +- Focus feedback on the contribution, not the contributor. +- No real credentials, secrets, or PII committed to YAML — use install-form fields or `consts:` placeholders. +- Report concerns via GitHub Issues. --- -Thank you for contributing! - +Thanks for contributing. diff --git a/README.md b/README.md index 2185df9..b9d572e 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,10 @@

-

Elastic Workflow Library

+

Elastic Workflow Template Library

- A curated collection of workflows for the Elastic platform, covering security, observability and search examples. + Source repo for the Workflow Template Library that ships in Kibana.

@@ -21,587 +21,132 @@

- Elastic 9.3+  - 57 Workflows  + Kibana 9.5+  + Tech Preview  YAML  - Liquid  Apache 2.0  Slack

--- -## Table of Contents - -- [Overview](#overview) -- [Quick Start](#quick-start) -- [Repository Structure](#repository-structure) -- [Workflow Categories](#workflow-categories) -- [Workflow Schema](#workflow-schema) -- [Key Concepts](#key-concepts) - - [Triggers](#triggers) - - [Variable Syntax](#variable-syntax) - - [Liquid Templating](#liquid-templating) - - [Error Handling](#error-handling) -- [Importing Workflows](#importing-workflows) -- [Examples](#examples) -- [Contributing](#contributing) -- [License](#license) - ---- - ## Overview -This repository contains **57 workflows** designed for use with Elastic Workflows, a platform feature for automating operations across the Elastic Stack. These workflows cover a wide range of use cases: - -| Category | Description | -|----------|-------------| -| **Security** | Threat detection, incident response, enrichment, and hunting | -| **Observability** | Monitoring, log analysis, and root cause analysis | -| **Search** | Elasticsearch queries, ES\|QL, semantic search | -| **Integrations** | Splunk, Slack, Jenkins, JIRA, Caldera, and more | -| **AI Agents** | Agentic workflows and AI-powered automation | -| **Data** | ETL, ingestion, and document management | - -### What are Elastic Workflows? +This repo holds the source of the **Workflow Template Library** — a curated catalogue of installable, parameterised workflow templates that Kibana users browse and install from the Workflows app. -Elastic Workflows provide a declarative YAML-based approach to automating operations across the Elastic platform. They integrate natively with: +Each template is a YAML file that combines: -- **Elasticsearch** - Query, aggregate, and index data with ES|QL and DSL -- **Kibana** - Create cases, manage alerts, interact with Security and Observability features -- **External Systems** - Splunk, Slack, Jenkins, JIRA, and any HTTP API -- **AI/ML** - Integrate with language models for intelligent analysis and agents +- A `template-metadata` header describing the template to Kibana (name, description, version, supported Kibana versions, categories, optional install form). +- A standard workflow body (`consts:`, `inputs:` / `triggers:`, `steps:`) that runs once installed. -### Key Features - -- **Declarative YAML** - Define what you want, not how to do it -- **Triggers** - Manual, scheduled, or alert-driven -- **Extensible** - Connect to any HTTP API or Elastic feature -- **Version Control** - Store workflows as code, track changes in Git -- **Shareable** - Import/export workflows between environments +The build pipeline in this repo turns the source templates into per-Kibana-version catalogues and uploads them to a CDN. Kibana fetches the catalogue at install time, renders the install form, substitutes the operator's values, and persists the resulting workflow as a Kibana saved object. --- -## Quick Start - -### 1. Browse Workflows - -Explore the [`workflows/`](./workflows) directory organized by use case: - -``` -workflows/ -├── security/ # Security operations -│ ├── detection/ # Alert management, threat detection -│ ├── response/ # Incident response, case management -│ ├── enrichment/ # Threat intel, IP/hash lookups -│ └── hunting/ # Threat hunting queries -├── integrations/ # Third-party integrations -│ ├── splunk/ # Splunk queries and enrichment -│ ├── slack/ # Channel management, notifications -│ ├── jenkins/ # CI/CD automation -│ ├── jira/ # Ticket management -│ ├── caldera/ # Adversary emulation -│ ├── firebase/ # Authentication -│ └── snowflake/ # Data warehouse queries -├── search/ # Search and query workflows -├── observability/ # Monitoring and analysis -├── ai-agents/ # AI-powered automation -├── data/ # ETL and data management -├── utilities/ # Common utility workflows -└── examples/ # Demo and getting-started -``` - -### 2. Review and Customize - -Each workflow includes inline comments explaining every section: - -```yaml -# ============================================================================= -# Workflow: IP Reputation Check -# Category: security/enrichment -# -# Assess the reputation of a given IP address using threat intelligence -# ============================================================================= - -name: IP Reputation Check - -# CONSTANTS - Update these values for your environment -consts: - abuseipdb_api_key: YOUR-API-KEY-HERE # Get from AbuseIPDB +## Repository structure -# INPUTS - Parameters provided at runtime -inputs: - - name: ip_address - type: string - required: true ``` - -### 3. Import into Kibana - -**Option A: Kibana UI** -1. Navigate to **Management → Workflows** in Kibana -2. Click **Create workflow** -3. Paste the YAML content -4. Save and test - -**Option B: API Import** -```bash -curl -X POST "https://your-kibana-url/api/workflows" \ - -H "kbn-xsrf: true" \ - -H "x-elastic-internal-origin: Kibana" \ - -H "Content-Type: application/json" \ - -H "Authorization: ApiKey YOUR_API_KEY" \ - -d '{"yaml": "'"$(cat workflows/security/enrichment/ip-reputation-check.yaml)"'"}' +elastic/workflows/ +├── library/ +│ ├── workflows/ # one directory per template, slug-matched +│ │ ├── ip-reputation-check/ +│ │ │ └── ip-reputation-check.yaml +│ │ └── … +│ └── categories.yaml # closed-vocab category registry +├── kibana-versions.json # policy file (latest, oldest, cataloguePer) +├── scripts/ +│ └── build-catalog.mjs # catalogue generator (Node 20+, ESM) +├── docs/ +│ ├── concepts.md # workflow engine concepts +│ ├── schema.md # workflow YAML schema reference +│ └── importing.md # raw-YAML import paths (for local dev) +├── CONTRIBUTING.md # template authoring guide +├── package.json +└── README.md ``` -See [docs/importing.md](./docs/importing.md) for detailed instructions. - ---- - -## Repository Structure - -``` -elastic-workflows/ -├── README.md # This file -├── CONTRIBUTING.md # Contribution guidelines -├── LICENSE.txt # Apache 2.0 license -├── workflows/ # All workflow YAML files -│ ├── security/ # Security operations -│ │ ├── detection/ # Threat detection workflows -│ │ ├── response/ # Incident response workflows -│ │ ├── enrichment/ # Enrichment workflows -│ │ └── hunting/ # Threat hunting workflows -│ ├── integrations/ # Third-party integrations -│ │ ├── splunk/ -│ │ ├── slack/ -│ │ ├── jenkins/ -│ │ ├── jira/ -│ │ ├── caldera/ -│ │ ├── firebase/ -│ │ └── snowflake/ -│ ├── search/ # Search workflows -│ ├── observability/ # Observability workflows -│ ├── ai-agents/ # AI agent workflows -│ ├── data/ # Data/ETL workflows -│ ├── utilities/ # Utility workflows -│ └── examples/ # Demo workflows -└── docs/ # Extended documentation - ├── schema.md # Complete YAML schema reference - ├── concepts.md # Workflow concepts explained - └── importing.md # Import instructions -``` +`library/` is the source. `dist/v1/` is the build output (gitignored; produced by `npm run build:catalog`). --- -## Workflow Categories - -### Security - -Workflows for security operations, threat detection, and incident response. - -| Category | Count | Description | -|----------|-------|-------------| -| [security/detection](./workflows/security/detection/) | 8 | Alert management, threat detection, rule execution | -| [security/enrichment](./workflows/security/enrichment/) | 5 | VirusTotal, IP reputation, threat intel lookups | -| [security/response](./workflows/security/response/) | 4 | Incident response, triage, and case management | - -### Integrations - -Workflows for connecting Elastic with external systems. - -| Category | Count | Description | -|----------|-------|-------------| -| [integrations/splunk](./workflows/integrations/splunk/) | 5 | Splunk queries, enrichment, data retrieval | -| [integrations/caldera](./workflows/integrations/caldera/) | 4 | MITRE Caldera adversary emulation | -| [integrations/slack](./workflows/integrations/slack/) | 3 | Channel creation, user management, notifications | -| [integrations/firebase](./workflows/integrations/firebase/) | 2 | Firebase authentication | -| [integrations/jenkins](./workflows/integrations/jenkins/) | 1 | CI/CD build automation | -| [integrations/jira](./workflows/integrations/jira/) | 1 | Ticket creation | -| [integrations/snowflake](./workflows/integrations/snowflake/) | 1 | Data warehouse queries | - -### Platform Features - -| Category | Count | Description | -|----------|-------|-------------| -| [search](./workflows/search/) | 4 | ES\|QL, semantic search, web search | -| [observability](./workflows/observability/) | 1 | Monitoring, log analysis, AI-powered observability | -| [ai-agents](./workflows/ai-agents/) | 2 | AI agent invocation and automation | -| [data](./workflows/data/) | 3 | ETL, ingestion, document management | -| [utilities](./workflows/utilities/) | 11 | Common operations and helpers | -| [examples](./workflows/examples/) | 2 | Getting started demos | +## Template format -### Featured Workflows - -| Workflow | Category | Description | -|----------|----------|-------------| -| [IP Reputation Check](./workflows/security/enrichment/ip-reputation-check.yaml) | Security | Check IP against AbuseIPDB and geolocation | -| [Hash Threat Check](./workflows/security/detection/hash-threat-check.yaml) | Security | VirusTotal file hash analysis | -| [Splunk Query](./workflows/integrations/splunk/splunk-query.yaml) | Integration | Execute Splunk searches | -| [Create Slack Channel](./workflows/integrations/slack/create-slack-channel.yaml) | Integration | Automated Slack channel creation | -| [Semantic Knowledge Search](./workflows/search/semantic-knowledge-search.yaml) | Search | AI-powered semantic search | -| [AD Automated Triaging](./workflows/security/response/ad-automated-triaging.yaml) | Security | Automated security alert triage workflow | - ---- - -## Workflow Schema - -Every workflow follows a consistent YAML schema: +A minimal example: ```yaml -# Required fields -name: "Workflow Name" # Human-readable name -steps: # At least one step required - - name: "Step Name" # Step identifier - type: "action.type" # Action to perform - with: # Action parameters - key: value - -# Optional fields -description: "What this does" # Detailed description -tags: # Categories for organization - - observability - - search -triggers: # How the workflow is invoked - - type: scheduled - with: - every: "1d" # Daily -consts: # Reusable constants - api_key: "value" -inputs: # Runtime parameters - - name: query - type: string - required: true -``` - -### Common Action Types - -| Action | Description | Use Case | -|--------|-------------|----------| -| `http` | HTTP requests | API calls, webhooks | -| `elasticsearch.search` | Search ES indices | Data retrieval | -| `elasticsearch.index` | Index documents | Data storage | -| `kibana.cases` | Case management | Incident response | -| `kibana.alert` | Alert operations | Detection | -| `console` | Log output | Debugging | -| `foreach` | Loop over arrays | Batch processing | +template-metadata: + slug: ip-reputation-check + version: "1.0.0" + availability: ">=9.5.0" + name: "IP Reputation Check (AbuseIPDB)" + description: "Assess the reputation of an IP address using AbuseIPDB." + solutions: [security] # optional; omit for cross-solution + categories: [enrichment, threat-intel] # closed vocab; entries from library/categories.yaml + icon: abuseipdb # optional + install: # only when the body uses __install__. + form: + - name: abuseipdb-connector + label: "AbuseIPDB connector" + inputType: connector + connectorType: .abuseipdb + required: true -See [docs/schema.md](./docs/schema.md) for the complete schema reference. - ---- - -## Key Concepts - -### Triggers - -Workflows support multiple trigger types: +name: IP Reputation Check +description: Check IP reputation via AbuseIPDB. -```yaml -# Manual (on-demand) triggers: - type: manual + inputs: + - name: ip_address + type: string + required: true -# Scheduled (simple interval) -triggers: - - type: scheduled - with: - every: "6h" # Every 6 hours - -# Alert-driven -triggers: - - type: alert -``` - -### Variable Syntax - -Reference values using double curly braces: - -```yaml -# Constants -url: "{{ consts.api_url }}/endpoint" - -# Inputs -query: "host.ip: {{ inputs.target_ip }}" - -# Step outputs -message: "Found {{ steps.search.output.hits.total }} results" -``` - -### Liquid Templating - -Workflows support [Liquid](https://shopify.github.io/liquid/) templating for dynamic content. Use filters to transform data inline. - -#### Common Filters - -| Filter | Description | Example | -|--------|-------------|---------| -| `json` | Convert to JSON string | `{{ object \| json }}` | -| `json_parse` | Parse JSON string to object | `{{ json_string \| json_parse }}` | -| `size` | Get array length or string length | `{{ items \| size }}` | -| `first` / `last` | Get first/last array item | `{{ items \| first }}` | -| `map` | Extract property from array | `{{ users \| map: "name" }}` | -| `where` | Filter array by property | `{{ items \| where: "status", "active" }}` | -| `where_exp` | Filter with expression | `{{ items \| where_exp: "item.price > 100" }}` | -| `join` | Join array to string | `{{ tags \| join: ", " }}` | -| `split` | Split string to array | `{{ csv \| split: "," }}` | -| `default` | Fallback value | `{{ name \| default: "Unknown" }}` | -| `date` | Format date | `{{ "now" \| date: "%Y-%m-%d" }}` | -| `upcase` / `downcase` | Change case | `{{ text \| upcase }}` | -| `strip` | Remove whitespace | `{{ text \| strip }}` | -| `replace` | Replace substring | `{{ text \| replace: "old", "new" }}` | -| `truncate` | Shorten string | `{{ text \| truncate: 50 }}` | -| `base64_encode` / `base64_decode` | Base64 encoding | `{{ text \| base64_encode }}` | -| `url_encode` / `url_decode` | URL encoding | `{{ text \| url_encode }}` | - -#### Array Manipulation - -```yaml -# Filter products where price > 100 -{{ products | where_exp: "item.price > 100" }} - -# Find first matching item -{{ products | find: "type", "book" }} - -# Check if any item matches -{{ products | has: "category", "electronics" }} - -# Remove items matching condition -{{ products | reject_exp: "item.stock == 0" }} - -# Sort by property -{{ products | sort: "name" }} - -# Get unique values -{{ items | uniq }} - -# Concatenate arrays -{{ array1 | concat: array2 }} -``` - -#### String Operations - -```yaml -# Format message with data -message: "Alert: {{ event.rule.name | upcase }} on {{ event.host.name }}" - -# Build URL with encoding -url: "https://api.example.com/search?q={{ query | url_encode }}" - -# Extract substring -short_hash: "{{ file.hash.sha256 | slice: 0, 8 }}" - -# Default values for missing data -user: "{{ event.user.name | default: 'unknown' }}" -``` - -#### Control Flow - -Use Liquid tags for conditional logic and loops: - -```yaml -message: | - {%- if steps.search.output.hits.total > 0 -%} - Found {{ steps.search.output.hits.total }} results - {%- else -%} - No results found - {%- endif -%} -``` - -```yaml -# Loop over items -message: | - {%- for alert in event.alerts -%} - - {{ alert.rule.name }}: {{ alert.severity }} - {%- endfor -%} -``` - -```yaml -# Assign variables -message: | - {%- assign severity = event.alerts[0].severity -%} - {%- case severity -%} - {%- when "critical" -%} - 🔴 CRITICAL: Immediate action required - {%- when "high" -%} - 🟠 HIGH: Investigate promptly - {%- else -%} - 🟢 Normal priority - {%- endcase -%} -``` - -
-All Supported Filters (click to expand) - -**Math**: `abs`, `at_least`, `at_most`, `ceil`, `divided_by`, `floor`, `minus`, `modulo`, `plus`, `round`, `times` - -**String**: `append`, `capitalize`, `downcase`, `escape`, `lstrip`, `prepend`, `remove`, `remove_first`, `remove_last`, `replace`, `replace_first`, `replace_last`, `rstrip`, `slice`, `split`, `strip`, `strip_html`, `strip_newlines`, `truncate`, `truncatewords`, `upcase` - -**Array**: `compact`, `concat`, `first`, `group_by`, `group_by_exp`, `join`, `last`, `map`, `pop`, `push`, `reverse`, `shift`, `size`, `sort`, `sort_natural`, `uniq`, `unshift`, `where`, `where_exp`, `find`, `find_exp`, `has`, `has_exp`, `reject`, `reject_exp` - -**Date**: `date`, `date_to_long_string`, `date_to_rfc822`, `date_to_string`, `date_to_xmlschema` - -**Encoding**: `base64_decode`, `base64_encode`, `cgi_escape`, `uri_escape`, `url_decode`, `url_encode`, `xml_escape`, `json`, `json_parse` - -**Utility**: `default`, `escape_once`, `normalize_whitespace`, `number_of_words`, `slugify`, `array_to_sentence_string` - -
- -### Error Handling - -```yaml steps: - - name: api_call - type: http + - name: check_abuseipdb + type: abuseipdb.checkIp + connector-id: __install__.abuseipdb-connector with: - url: "{{ consts.api_url }}" - on-failure: - retry: - max-attempts: 3 - delay: 5s - continue: true # Proceed even on failure + ipAddress: "{{ inputs.ip_address }}" ``` -See [docs/concepts.md](./docs/concepts.md) for detailed explanations. +See [CONTRIBUTING.md](./CONTRIBUTING.md) for the full authoring guide — required vs optional fields, the `install.form` discipline, categories vocabulary rules, step-type conventions, versioning, and local validation. --- -## Importing Workflows - -### Kibana UI - -1. Open Kibana → **Management → Workflows** -2. Click **Create workflow** -3. Paste YAML content -4. Update constants for your environment -5. Save - -### API +## Integration with Kibana -```bash -cat workflow.yaml | jq -Rs '{yaml: .}' | \ - curl -X POST "https://KIBANA_URL/api/workflows" \ - -H "kbn-xsrf: true" \ - -H "x-elastic-internal-origin: Kibana" \ - -H "Content-Type: application/json" \ - -H "Authorization: ApiKey API_KEY" \ - -d @- -``` +In Kibana 9.5+ (Tech Preview), the Workflows app reads the published catalogue from the CDN and renders a browser of installable templates. Installing a template prompts the operator for the values declared in `install.form`, substitutes them for the `__install__.` placeholders in the body, and persists the resulting workflow as a saved object — at which point it runs like any other workflow. -### Bulk Import +Consumers see: -```bash -for file in workflows/security/**/*.yaml; do - echo "Importing: $file" - cat "$file" | jq -Rs '{yaml: .}' | \ - curl -s -X POST "https://KIBANA_URL/api/workflows" \ - -H "kbn-xsrf: true" \ - -H "x-elastic-internal-origin: Kibana" \ - -H "Content-Type: application/json" \ - -H "Authorization: ApiKey API_KEY" \ - -d @- -done -``` +- `/v1/kibana-versions.json` — the resolved list of available catalogues. +- `/v1//catalogs/templates.json` — the catalogue rows for a given Kibana version. +- `/v1/templates//.yaml` — immutable, version-keyed template bodies. -See [docs/importing.md](./docs/importing.md) for complete instructions. +The catalogue is republished on every merge to `main`. --- -## Examples - -### Example 1: ES|QL Query - -Query Elasticsearch with ES|QL and process results: - -```yaml -name: ES|QL Query Example -triggers: - - type: manual - -inputs: - - name: query - type: string - default: "FROM logs-* | LIMIT 10" +## Building the catalogue locally -steps: - - name: execute_query - type: elasticsearch.esql.query - with: - format: json - query: "{{ inputs.query }}" - - - name: store_count - type: data.set - with: - row_count: "{{ steps.execute_query.output.values | size }}" +```bash +npm install +npm run build:catalog ``` -### Example 2: Scheduled Report Workflow - -Run an ES|QL query and send a summary to Slack: - -```yaml -name: Daily Security Summary -triggers: - - type: scheduled - with: - every: "1d" # Daily - -consts: - slack_webhook: "https://hooks.slack.com/..." - -steps: - - name: query_alerts - type: elasticsearch.esql.query - with: - format: json - query: | - FROM .alerts-security.alerts-default - | WHERE @timestamp > NOW() - 24 hours - | STATS alert_count = COUNT(*) BY host.name - | SORT alert_count DESC - | LIMIT 10 - - - name: notify_slack - type: http - with: - url: "{{ consts.slack_webhook }}" - method: POST - body: - text: "🔔 Daily Alert Summary: {{ steps.query_alerts.output.values | size }} hosts with alerts in the last 24h" -``` +Outputs to `dist/v1/`. The script fetches the live Kibana `main` semver and the list of supported named minors from `elastic/kibana`. For offline iteration, two env-var overrides skip the network calls — see the [Validating locally](./CONTRIBUTING.md#validating-locally) section of CONTRIBUTING.md. --- -## Contributing - -We welcome contributions! See [CONTRIBUTING.md](./CONTRIBUTING.md) for guidelines. +## Further reading -### How to Contribute - -1. Fork this repository -2. Add your workflow to the appropriate category -3. Include inline comments explaining each step -4. Test in a Kibana environment -5. Submit a pull request - -### Workflow Guidelines - -- Include header comment block with description -- Add section comments (CONSTANTS, INPUTS, STEPS) -- Use meaningful step names -- Document required inputs and constants -- No hardcoded credentials +- [CONTRIBUTING.md](./CONTRIBUTING.md) — how to author or modify a template. +- [docs/concepts.md](./docs/concepts.md) — workflow engine concepts (triggers, steps, variables, Liquid, error handling). +- [docs/schema.md](./docs/schema.md) — workflow YAML schema reference. +- [docs/importing.md](./docs/importing.md) — raw-YAML import paths (Kibana UI / API / bulk), useful for local development before a template ships through the library. --- ## License -This project is licensed under the Apache License 2.0 - see [LICENSE](./LICENSE.txt) for details. - ---- - -## Resources - -- [Elastic Workflows Documentation](https://www.elastic.co/docs/explore-analyze/workflows) -- [ES|QL Reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/esql.html) -- [Kibana API Reference](https://www.elastic.co/guide/en/kibana/current/api.html) - +Apache 2.0 — see [LICENSE.txt](./LICENSE.txt). diff --git a/docs/importing.md b/docs/importing.md index c1ee79f..a2dba8c 100644 --- a/docs/importing.md +++ b/docs/importing.md @@ -73,7 +73,7 @@ The simplest way to import a single workflow. ### Example: Importing Hash Threat Check -1. Open [`workflows/security/detection/hash-threat-check.yaml`](../workflows/security/detection/hash-threat-check.yaml) +1. Open [`examples/security/detection/hash-threat-check.yaml`](../examples/security/detection/hash-threat-check.yaml) 2. Copy the content 3. In Kibana, create new workflow and paste 4. Update the VirusTotal API key: @@ -104,7 +104,7 @@ curl -X POST "https://YOUR_KIBANA_URL/api/workflows" \ ```bash # Read file and escape for JSON -YAML_CONTENT=$(cat workflows/detection/hash-threat-check.yaml) +YAML_CONTENT=$(cat examples/security/detection/hash-threat-check.yaml) curl -X POST "https://YOUR_KIBANA_URL/api/workflows" \ -H "kbn-xsrf: true" \ @@ -123,7 +123,7 @@ EOF ```bash # Install jq if not available: brew install jq (macOS) or apt install jq (Linux) -cat workflows/detection/hash-threat-check.yaml | \ +cat examples/security/detection/hash-threat-check.yaml | \ jq -Rs '{yaml: .}' | \ curl -X POST "https://YOUR_KIBANA_URL/api/workflows" \ -H "kbn-xsrf: true" \ @@ -239,7 +239,7 @@ Import multiple workflows at once. KIBANA_URL="${KIBANA_URL:-https://your-kibana-url}" API_KEY="${API_KEY:-your-api-key}" -WORKFLOW_DIR="${1:-workflows}" +WORKFLOW_DIR="${1:-examples}" import_workflow() { local file="$1" @@ -280,10 +280,10 @@ echo "Import complete!" ```bash # Import all workflows -./bulk_import.sh workflows/ +./bulk_import.sh examples/ # Import specific category -./bulk_import.sh workflows/detection/ +./bulk_import.sh examples/security/detection/ # With environment variables KIBANA_URL=https://my-kibana.example.com API_KEY=abc123 ./bulk_import.sh @@ -364,7 +364,7 @@ def bulk_import(directory): print(f"Import complete: {success} succeeded, {failed} failed") if __name__ == "__main__": - directory = sys.argv[1] if len(sys.argv) > 1 else "workflows" + directory = sys.argv[1] if len(sys.argv) > 1 else "examples" bulk_import(directory) ``` @@ -470,7 +470,7 @@ Error: YAML parsing failed **Solution:** Validate YAML with an online validator or `yamllint`: ```bash -yamllint workflows/detection/my-workflow.yaml +yamllint examples/security/detection/my-workflow.yaml ``` #### 2. Missing Required Fields diff --git a/workflows/ai-agents/README.md b/examples/ai-agents/README.md similarity index 100% rename from workflows/ai-agents/README.md rename to examples/ai-agents/README.md diff --git a/workflows/ai-agents/call-subagent-workflow.yaml b/examples/ai-agents/call-subagent-workflow.yaml similarity index 100% rename from workflows/ai-agents/call-subagent-workflow.yaml rename to examples/ai-agents/call-subagent-workflow.yaml diff --git a/workflows/ai-agents/invoke-an-agent.yaml b/examples/ai-agents/invoke-an-agent.yaml similarity index 100% rename from workflows/ai-agents/invoke-an-agent.yaml rename to examples/ai-agents/invoke-an-agent.yaml diff --git a/workflows/data/README.md b/examples/data/README.md similarity index 100% rename from workflows/data/README.md rename to examples/data/README.md diff --git a/workflows/data/getdocument.yaml b/examples/data/getdocument.yaml similarity index 100% rename from workflows/data/getdocument.yaml rename to examples/data/getdocument.yaml diff --git a/workflows/data/log-injection.yaml b/examples/data/log-injection.yaml similarity index 100% rename from workflows/data/log-injection.yaml rename to examples/data/log-injection.yaml diff --git a/workflows/data/rss-feed-ingest.yaml b/examples/data/rss-feed-ingest.yaml similarity index 100% rename from workflows/data/rss-feed-ingest.yaml rename to examples/data/rss-feed-ingest.yaml diff --git a/workflows/examples/README.md b/examples/examples/README.md similarity index 100% rename from workflows/examples/README.md rename to examples/examples/README.md diff --git a/workflows/examples/national-parks-demo.yaml b/examples/examples/national-parks-demo.yaml similarity index 100% rename from workflows/examples/national-parks-demo.yaml rename to examples/examples/national-parks-demo.yaml diff --git a/workflows/integrations/amazon-s3/README.md b/examples/integrations/amazon-s3/README.md similarity index 100% rename from workflows/integrations/amazon-s3/README.md rename to examples/integrations/amazon-s3/README.md diff --git a/workflows/integrations/amazon-s3/download-file.yaml b/examples/integrations/amazon-s3/download-file.yaml similarity index 100% rename from workflows/integrations/amazon-s3/download-file.yaml rename to examples/integrations/amazon-s3/download-file.yaml diff --git a/workflows/integrations/caldera/README.md b/examples/integrations/caldera/README.md similarity index 100% rename from workflows/integrations/caldera/README.md rename to examples/integrations/caldera/README.md diff --git a/workflows/integrations/caldera/create-caldera-ability.yaml b/examples/integrations/caldera/create-caldera-ability.yaml similarity index 100% rename from workflows/integrations/caldera/create-caldera-ability.yaml rename to examples/integrations/caldera/create-caldera-ability.yaml diff --git a/workflows/integrations/caldera/create-caldera-adversary.yaml b/examples/integrations/caldera/create-caldera-adversary.yaml similarity index 100% rename from workflows/integrations/caldera/create-caldera-adversary.yaml rename to examples/integrations/caldera/create-caldera-adversary.yaml diff --git a/workflows/integrations/caldera/create-caldera-operation.yaml b/examples/integrations/caldera/create-caldera-operation.yaml similarity index 100% rename from workflows/integrations/caldera/create-caldera-operation.yaml rename to examples/integrations/caldera/create-caldera-operation.yaml diff --git a/workflows/integrations/caldera/get-caldera-operation-report.yaml b/examples/integrations/caldera/get-caldera-operation-report.yaml similarity index 100% rename from workflows/integrations/caldera/get-caldera-operation-report.yaml rename to examples/integrations/caldera/get-caldera-operation-report.yaml diff --git a/workflows/integrations/confluence-cloud/README.md b/examples/integrations/confluence-cloud/README.md similarity index 100% rename from workflows/integrations/confluence-cloud/README.md rename to examples/integrations/confluence-cloud/README.md diff --git a/workflows/integrations/confluence-cloud/get-resource.yaml b/examples/integrations/confluence-cloud/get-resource.yaml similarity index 100% rename from workflows/integrations/confluence-cloud/get-resource.yaml rename to examples/integrations/confluence-cloud/get-resource.yaml diff --git a/workflows/integrations/confluence-cloud/list-resource.yaml b/examples/integrations/confluence-cloud/list-resource.yaml similarity index 100% rename from workflows/integrations/confluence-cloud/list-resource.yaml rename to examples/integrations/confluence-cloud/list-resource.yaml diff --git a/workflows/integrations/firebase/README.md b/examples/integrations/firebase/README.md similarity index 100% rename from workflows/integrations/firebase/README.md rename to examples/integrations/firebase/README.md diff --git a/workflows/integrations/firebase/grant-and-fetch-firebase-bearer-token.yaml b/examples/integrations/firebase/grant-and-fetch-firebase-bearer-token.yaml similarity index 100% rename from workflows/integrations/firebase/grant-and-fetch-firebase-bearer-token.yaml rename to examples/integrations/firebase/grant-and-fetch-firebase-bearer-token.yaml diff --git a/workflows/integrations/firebase/grant-firebase-bearer-token.yaml b/examples/integrations/firebase/grant-firebase-bearer-token.yaml similarity index 100% rename from workflows/integrations/firebase/grant-firebase-bearer-token.yaml rename to examples/integrations/firebase/grant-firebase-bearer-token.yaml diff --git a/workflows/integrations/firecrawl/README.md b/examples/integrations/firecrawl/README.md similarity index 100% rename from workflows/integrations/firecrawl/README.md rename to examples/integrations/firecrawl/README.md diff --git a/workflows/integrations/firecrawl/crawl.yaml b/examples/integrations/firecrawl/crawl.yaml similarity index 100% rename from workflows/integrations/firecrawl/crawl.yaml rename to examples/integrations/firecrawl/crawl.yaml diff --git a/workflows/integrations/github/README.md b/examples/integrations/github/README.md similarity index 100% rename from workflows/integrations/github/README.md rename to examples/integrations/github/README.md diff --git a/workflows/integrations/github/get-resource.yaml b/examples/integrations/github/get-resource.yaml similarity index 100% rename from workflows/integrations/github/get-resource.yaml rename to examples/integrations/github/get-resource.yaml diff --git a/workflows/integrations/github/list-resources.yaml b/examples/integrations/github/list-resources.yaml similarity index 100% rename from workflows/integrations/github/list-resources.yaml rename to examples/integrations/github/list-resources.yaml diff --git a/workflows/integrations/github/search.yaml b/examples/integrations/github/search.yaml similarity index 100% rename from workflows/integrations/github/search.yaml rename to examples/integrations/github/search.yaml diff --git a/workflows/integrations/google-calendar/README.md b/examples/integrations/google-calendar/README.md similarity index 100% rename from workflows/integrations/google-calendar/README.md rename to examples/integrations/google-calendar/README.md diff --git a/workflows/integrations/google-calendar/free-busy.yaml b/examples/integrations/google-calendar/free-busy.yaml similarity index 100% rename from workflows/integrations/google-calendar/free-busy.yaml rename to examples/integrations/google-calendar/free-busy.yaml diff --git a/workflows/integrations/google-calendar/get-event.yaml b/examples/integrations/google-calendar/get-event.yaml similarity index 100% rename from workflows/integrations/google-calendar/get-event.yaml rename to examples/integrations/google-calendar/get-event.yaml diff --git a/workflows/integrations/google-calendar/list-calendars.yaml b/examples/integrations/google-calendar/list-calendars.yaml similarity index 100% rename from workflows/integrations/google-calendar/list-calendars.yaml rename to examples/integrations/google-calendar/list-calendars.yaml diff --git a/workflows/integrations/google-calendar/list-events.yaml b/examples/integrations/google-calendar/list-events.yaml similarity index 100% rename from workflows/integrations/google-calendar/list-events.yaml rename to examples/integrations/google-calendar/list-events.yaml diff --git a/workflows/integrations/google-calendar/search.yaml b/examples/integrations/google-calendar/search.yaml similarity index 100% rename from workflows/integrations/google-calendar/search.yaml rename to examples/integrations/google-calendar/search.yaml diff --git a/workflows/integrations/google-cloud-storage/README.md b/examples/integrations/google-cloud-storage/README.md similarity index 100% rename from workflows/integrations/google-cloud-storage/README.md rename to examples/integrations/google-cloud-storage/README.md diff --git a/workflows/integrations/google-cloud-storage/download.yaml b/examples/integrations/google-cloud-storage/download.yaml similarity index 100% rename from workflows/integrations/google-cloud-storage/download.yaml rename to examples/integrations/google-cloud-storage/download.yaml diff --git a/workflows/integrations/google-cloud-storage/get-object-metadata.yaml b/examples/integrations/google-cloud-storage/get-object-metadata.yaml similarity index 100% rename from workflows/integrations/google-cloud-storage/get-object-metadata.yaml rename to examples/integrations/google-cloud-storage/get-object-metadata.yaml diff --git a/workflows/integrations/google-cloud-storage/list-buckets.yaml b/examples/integrations/google-cloud-storage/list-buckets.yaml similarity index 100% rename from workflows/integrations/google-cloud-storage/list-buckets.yaml rename to examples/integrations/google-cloud-storage/list-buckets.yaml diff --git a/workflows/integrations/google-cloud-storage/list-objects.yaml b/examples/integrations/google-cloud-storage/list-objects.yaml similarity index 100% rename from workflows/integrations/google-cloud-storage/list-objects.yaml rename to examples/integrations/google-cloud-storage/list-objects.yaml diff --git a/workflows/integrations/google-cloud-storage/list-projects.yaml b/examples/integrations/google-cloud-storage/list-projects.yaml similarity index 100% rename from workflows/integrations/google-cloud-storage/list-projects.yaml rename to examples/integrations/google-cloud-storage/list-projects.yaml diff --git a/workflows/integrations/google-drive/README.md b/examples/integrations/google-drive/README.md similarity index 100% rename from workflows/integrations/google-drive/README.md rename to examples/integrations/google-drive/README.md diff --git a/workflows/integrations/google-drive/download.yaml b/examples/integrations/google-drive/download.yaml similarity index 100% rename from workflows/integrations/google-drive/download.yaml rename to examples/integrations/google-drive/download.yaml diff --git a/workflows/integrations/jenkins/README.md b/examples/integrations/jenkins/README.md similarity index 100% rename from workflows/integrations/jenkins/README.md rename to examples/integrations/jenkins/README.md diff --git a/workflows/integrations/jenkins/trigger-jenkins-build-simple.yaml b/examples/integrations/jenkins/trigger-jenkins-build-simple.yaml similarity index 100% rename from workflows/integrations/jenkins/trigger-jenkins-build-simple.yaml rename to examples/integrations/jenkins/trigger-jenkins-build-simple.yaml diff --git a/workflows/integrations/jira-cloud/README.md b/examples/integrations/jira-cloud/README.md similarity index 100% rename from workflows/integrations/jira-cloud/README.md rename to examples/integrations/jira-cloud/README.md diff --git a/workflows/integrations/jira-cloud/get-resource.yaml b/examples/integrations/jira-cloud/get-resource.yaml similarity index 100% rename from workflows/integrations/jira-cloud/get-resource.yaml rename to examples/integrations/jira-cloud/get-resource.yaml diff --git a/workflows/integrations/jira/README.md b/examples/integrations/jira/README.md similarity index 100% rename from workflows/integrations/jira/README.md rename to examples/integrations/jira/README.md diff --git a/workflows/integrations/jira/create-jira-ticket.yaml b/examples/integrations/jira/create-jira-ticket.yaml similarity index 100% rename from workflows/integrations/jira/create-jira-ticket.yaml rename to examples/integrations/jira/create-jira-ticket.yaml diff --git a/workflows/integrations/microsoft-teams/README.md b/examples/integrations/microsoft-teams/README.md similarity index 100% rename from workflows/integrations/microsoft-teams/README.md rename to examples/integrations/microsoft-teams/README.md diff --git a/workflows/integrations/microsoft-teams/list-resources.yaml b/examples/integrations/microsoft-teams/list-resources.yaml similarity index 100% rename from workflows/integrations/microsoft-teams/list-resources.yaml rename to examples/integrations/microsoft-teams/list-resources.yaml diff --git a/workflows/integrations/microsoft-teams/search-messages.yaml b/examples/integrations/microsoft-teams/search-messages.yaml similarity index 100% rename from workflows/integrations/microsoft-teams/search-messages.yaml rename to examples/integrations/microsoft-teams/search-messages.yaml diff --git a/workflows/integrations/pagerduty/README.md b/examples/integrations/pagerduty/README.md similarity index 100% rename from workflows/integrations/pagerduty/README.md rename to examples/integrations/pagerduty/README.md diff --git a/workflows/integrations/pagerduty/get-by-id.yaml b/examples/integrations/pagerduty/get-by-id.yaml similarity index 100% rename from workflows/integrations/pagerduty/get-by-id.yaml rename to examples/integrations/pagerduty/get-by-id.yaml diff --git a/workflows/integrations/pagerduty/get-escalation-policies.yaml b/examples/integrations/pagerduty/get-escalation-policies.yaml similarity index 100% rename from workflows/integrations/pagerduty/get-escalation-policies.yaml rename to examples/integrations/pagerduty/get-escalation-policies.yaml diff --git a/workflows/integrations/pagerduty/get-incidents.yaml b/examples/integrations/pagerduty/get-incidents.yaml similarity index 100% rename from workflows/integrations/pagerduty/get-incidents.yaml rename to examples/integrations/pagerduty/get-incidents.yaml diff --git a/workflows/integrations/pagerduty/get-oncalls.yaml b/examples/integrations/pagerduty/get-oncalls.yaml similarity index 100% rename from workflows/integrations/pagerduty/get-oncalls.yaml rename to examples/integrations/pagerduty/get-oncalls.yaml diff --git a/workflows/integrations/pagerduty/get-schedules.yaml b/examples/integrations/pagerduty/get-schedules.yaml similarity index 100% rename from workflows/integrations/pagerduty/get-schedules.yaml rename to examples/integrations/pagerduty/get-schedules.yaml diff --git a/workflows/integrations/pagerduty/search.yaml b/examples/integrations/pagerduty/search.yaml similarity index 100% rename from workflows/integrations/pagerduty/search.yaml rename to examples/integrations/pagerduty/search.yaml diff --git a/workflows/integrations/servicenow/README.md b/examples/integrations/servicenow/README.md similarity index 100% rename from workflows/integrations/servicenow/README.md rename to examples/integrations/servicenow/README.md diff --git a/workflows/integrations/servicenow/get-attachment.yaml b/examples/integrations/servicenow/get-attachment.yaml similarity index 100% rename from workflows/integrations/servicenow/get-attachment.yaml rename to examples/integrations/servicenow/get-attachment.yaml diff --git a/workflows/integrations/servicenow/get-record-with-comments.yaml b/examples/integrations/servicenow/get-record-with-comments.yaml similarity index 100% rename from workflows/integrations/servicenow/get-record-with-comments.yaml rename to examples/integrations/servicenow/get-record-with-comments.yaml diff --git a/workflows/integrations/sharepoint-online/README.md b/examples/integrations/sharepoint-online/README.md similarity index 100% rename from workflows/integrations/sharepoint-online/README.md rename to examples/integrations/sharepoint-online/README.md diff --git a/workflows/integrations/sharepoint-online/download.yaml b/examples/integrations/sharepoint-online/download.yaml similarity index 100% rename from workflows/integrations/sharepoint-online/download.yaml rename to examples/integrations/sharepoint-online/download.yaml diff --git a/workflows/integrations/sharepoint-online/list-resources.yaml b/examples/integrations/sharepoint-online/list-resources.yaml similarity index 100% rename from workflows/integrations/sharepoint-online/list-resources.yaml rename to examples/integrations/sharepoint-online/list-resources.yaml diff --git a/workflows/integrations/sharepoint-server/README.md b/examples/integrations/sharepoint-server/README.md similarity index 100% rename from workflows/integrations/sharepoint-server/README.md rename to examples/integrations/sharepoint-server/README.md diff --git a/workflows/integrations/sharepoint-server/download.yaml b/examples/integrations/sharepoint-server/download.yaml similarity index 100% rename from workflows/integrations/sharepoint-server/download.yaml rename to examples/integrations/sharepoint-server/download.yaml diff --git a/workflows/integrations/sharepoint-server/list-resources.yaml b/examples/integrations/sharepoint-server/list-resources.yaml similarity index 100% rename from workflows/integrations/sharepoint-server/list-resources.yaml rename to examples/integrations/sharepoint-server/list-resources.yaml diff --git a/workflows/integrations/slack/README.md b/examples/integrations/slack/README.md similarity index 100% rename from workflows/integrations/slack/README.md rename to examples/integrations/slack/README.md diff --git a/workflows/integrations/slack/add-users-to-slack-channel.yaml b/examples/integrations/slack/add-users-to-slack-channel.yaml similarity index 100% rename from workflows/integrations/slack/add-users-to-slack-channel.yaml rename to examples/integrations/slack/add-users-to-slack-channel.yaml diff --git a/workflows/integrations/slack/create-slack-channel.yaml b/examples/integrations/slack/create-slack-channel.yaml similarity index 100% rename from workflows/integrations/slack/create-slack-channel.yaml rename to examples/integrations/slack/create-slack-channel.yaml diff --git a/workflows/integrations/slack/post-to-slack-channel-with-blocks-format.yaml b/examples/integrations/slack/post-to-slack-channel-with-blocks-format.yaml similarity index 100% rename from workflows/integrations/slack/post-to-slack-channel-with-blocks-format.yaml rename to examples/integrations/slack/post-to-slack-channel-with-blocks-format.yaml diff --git a/workflows/integrations/snowflake/README.md b/examples/integrations/snowflake/README.md similarity index 100% rename from workflows/integrations/snowflake/README.md rename to examples/integrations/snowflake/README.md diff --git a/workflows/integrations/snowflake/search-snowflake.yaml b/examples/integrations/snowflake/search-snowflake.yaml similarity index 100% rename from workflows/integrations/snowflake/search-snowflake.yaml rename to examples/integrations/snowflake/search-snowflake.yaml diff --git a/workflows/integrations/splunk/README.md b/examples/integrations/splunk/README.md similarity index 100% rename from workflows/integrations/splunk/README.md rename to examples/integrations/splunk/README.md diff --git a/workflows/integrations/splunk/inspect-splunk-fields.yaml b/examples/integrations/splunk/inspect-splunk-fields.yaml similarity index 100% rename from workflows/integrations/splunk/inspect-splunk-fields.yaml rename to examples/integrations/splunk/inspect-splunk-fields.yaml diff --git a/workflows/integrations/splunk/list-splunk-indices.yaml b/examples/integrations/splunk/list-splunk-indices.yaml similarity index 100% rename from workflows/integrations/splunk/list-splunk-indices.yaml rename to examples/integrations/splunk/list-splunk-indices.yaml diff --git a/workflows/integrations/splunk/search-splunk.yaml b/examples/integrations/splunk/search-splunk.yaml similarity index 100% rename from workflows/integrations/splunk/search-splunk.yaml rename to examples/integrations/splunk/search-splunk.yaml diff --git a/workflows/integrations/splunk/splunk-enrichment-with-virustotal-and-ai-analysis.yaml b/examples/integrations/splunk/splunk-enrichment-with-virustotal-and-ai-analysis.yaml similarity index 100% rename from workflows/integrations/splunk/splunk-enrichment-with-virustotal-and-ai-analysis.yaml rename to examples/integrations/splunk/splunk-enrichment-with-virustotal-and-ai-analysis.yaml diff --git a/workflows/integrations/splunk/splunk-query.yaml b/examples/integrations/splunk/splunk-query.yaml similarity index 100% rename from workflows/integrations/splunk/splunk-query.yaml rename to examples/integrations/splunk/splunk-query.yaml diff --git a/workflows/integrations/tavily/README.md b/examples/integrations/tavily/README.md similarity index 100% rename from workflows/integrations/tavily/README.md rename to examples/integrations/tavily/README.md diff --git a/workflows/integrations/tavily/search.yaml b/examples/integrations/tavily/search.yaml similarity index 100% rename from workflows/integrations/tavily/search.yaml rename to examples/integrations/tavily/search.yaml diff --git a/workflows/integrations/zendesk/README.md b/examples/integrations/zendesk/README.md similarity index 100% rename from workflows/integrations/zendesk/README.md rename to examples/integrations/zendesk/README.md diff --git a/workflows/integrations/zendesk/list-tickets.yaml b/examples/integrations/zendesk/list-tickets.yaml similarity index 100% rename from workflows/integrations/zendesk/list-tickets.yaml rename to examples/integrations/zendesk/list-tickets.yaml diff --git a/workflows/integrations/zendesk/search.yaml b/examples/integrations/zendesk/search.yaml similarity index 100% rename from workflows/integrations/zendesk/search.yaml rename to examples/integrations/zendesk/search.yaml diff --git a/workflows/integrations/zoom/README.md b/examples/integrations/zoom/README.md similarity index 100% rename from workflows/integrations/zoom/README.md rename to examples/integrations/zoom/README.md diff --git a/workflows/integrations/zoom/get-meeting.yaml b/examples/integrations/zoom/get-meeting.yaml similarity index 100% rename from workflows/integrations/zoom/get-meeting.yaml rename to examples/integrations/zoom/get-meeting.yaml diff --git a/workflows/integrations/zoom/list-resources.yaml b/examples/integrations/zoom/list-resources.yaml similarity index 100% rename from workflows/integrations/zoom/list-resources.yaml rename to examples/integrations/zoom/list-resources.yaml diff --git a/workflows/observability/README.md b/examples/observability/README.md similarity index 100% rename from workflows/observability/README.md rename to examples/observability/README.md diff --git a/workflows/observability/ai-steps-demo.yaml b/examples/observability/ai-steps-demo.yaml similarity index 100% rename from workflows/observability/ai-steps-demo.yaml rename to examples/observability/ai-steps-demo.yaml diff --git a/workflows/observability/root-cause-analysis-rca-workflow.yaml b/examples/observability/root-cause-analysis-rca-workflow.yaml similarity index 100% rename from workflows/observability/root-cause-analysis-rca-workflow.yaml rename to examples/observability/root-cause-analysis-rca-workflow.yaml diff --git a/workflows/search/README.md b/examples/search/README.md similarity index 100% rename from workflows/search/README.md rename to examples/search/README.md diff --git a/workflows/search/eql-to-esql.yaml b/examples/search/eql-to-esql.yaml similarity index 100% rename from workflows/search/eql-to-esql.yaml rename to examples/search/eql-to-esql.yaml diff --git a/workflows/search/es-ql-query-output-table-values-to-new-index.yaml b/examples/search/es-ql-query-output-table-values-to-new-index.yaml similarity index 100% rename from workflows/search/es-ql-query-output-table-values-to-new-index.yaml rename to examples/search/es-ql-query-output-table-values-to-new-index.yaml diff --git a/workflows/search/semantic-knowledge-search.yaml b/examples/search/semantic-knowledge-search.yaml similarity index 100% rename from workflows/search/semantic-knowledge-search.yaml rename to examples/search/semantic-knowledge-search.yaml diff --git a/workflows/search/web-search.yaml b/examples/search/web-search.yaml similarity index 100% rename from workflows/search/web-search.yaml rename to examples/search/web-search.yaml diff --git a/workflows/security/compliance/README.md b/examples/security/compliance/README.md similarity index 100% rename from workflows/security/compliance/README.md rename to examples/security/compliance/README.md diff --git a/workflows/security/compliance/pci-daily-assessment.yaml b/examples/security/compliance/pci-daily-assessment.yaml similarity index 100% rename from workflows/security/compliance/pci-daily-assessment.yaml rename to examples/security/compliance/pci-daily-assessment.yaml diff --git a/workflows/security/compliance/pci-violation-case-creator.yaml b/examples/security/compliance/pci-violation-case-creator.yaml similarity index 100% rename from workflows/security/compliance/pci-violation-case-creator.yaml rename to examples/security/compliance/pci-violation-case-creator.yaml diff --git a/workflows/security/detection/README.md b/examples/security/detection/README.md similarity index 100% rename from workflows/security/detection/README.md rename to examples/security/detection/README.md diff --git a/workflows/security/detection/add-alert-tag-fp.yaml b/examples/security/detection/add-alert-tag-fp.yaml similarity index 100% rename from workflows/security/detection/add-alert-tag-fp.yaml rename to examples/security/detection/add-alert-tag-fp.yaml diff --git a/workflows/security/detection/add-alert-tag-tp.yaml b/examples/security/detection/add-alert-tag-tp.yaml similarity index 100% rename from workflows/security/detection/add-alert-tag-tp.yaml rename to examples/security/detection/add-alert-tag-tp.yaml diff --git a/workflows/security/detection/create-alert-note.yaml b/examples/security/detection/create-alert-note.yaml similarity index 100% rename from workflows/security/detection/create-alert-note.yaml rename to examples/security/detection/create-alert-note.yaml diff --git a/workflows/security/detection/hash-threat-check.yaml b/examples/security/detection/hash-threat-check.yaml similarity index 100% rename from workflows/security/detection/hash-threat-check.yaml rename to examples/security/detection/hash-threat-check.yaml diff --git a/workflows/security/detection/manually-run-rules.yaml b/examples/security/detection/manually-run-rules.yaml similarity index 100% rename from workflows/security/detection/manually-run-rules.yaml rename to examples/security/detection/manually-run-rules.yaml diff --git a/workflows/security/detection/mark-alert-as-acknowledged.yaml b/examples/security/detection/mark-alert-as-acknowledged.yaml similarity index 100% rename from workflows/security/detection/mark-alert-as-acknowledged.yaml rename to examples/security/detection/mark-alert-as-acknowledged.yaml diff --git a/workflows/security/detection/mark-alert-as-closed.yaml b/examples/security/detection/mark-alert-as-closed.yaml similarity index 100% rename from workflows/security/detection/mark-alert-as-closed.yaml rename to examples/security/detection/mark-alert-as-closed.yaml diff --git a/workflows/security/detection/snmp-link-status-monitor.yaml b/examples/security/detection/snmp-link-status-monitor.yaml similarity index 100% rename from workflows/security/detection/snmp-link-status-monitor.yaml rename to examples/security/detection/snmp-link-status-monitor.yaml diff --git a/workflows/security/enrichment/README.md b/examples/security/enrichment/README.md similarity index 100% rename from workflows/security/enrichment/README.md rename to examples/security/enrichment/README.md diff --git a/workflows/security/enrichment/geoipfromdiscover.yaml b/examples/security/enrichment/geoipfromdiscover.yaml similarity index 100% rename from workflows/security/enrichment/geoipfromdiscover.yaml rename to examples/security/enrichment/geoipfromdiscover.yaml diff --git a/workflows/security/enrichment/ip-reputation-check.yaml b/examples/security/enrichment/ip-reputation-check.yaml similarity index 100% rename from workflows/security/enrichment/ip-reputation-check.yaml rename to examples/security/enrichment/ip-reputation-check.yaml diff --git a/workflows/security/enrichment/rootcausefromdiscover.yaml b/examples/security/enrichment/rootcausefromdiscover.yaml similarity index 100% rename from workflows/security/enrichment/rootcausefromdiscover.yaml rename to examples/security/enrichment/rootcausefromdiscover.yaml diff --git a/workflows/security/enrichment/send-hash-to-virustotal.yaml b/examples/security/enrichment/send-hash-to-virustotal.yaml similarity index 100% rename from workflows/security/enrichment/send-hash-to-virustotal.yaml rename to examples/security/enrichment/send-hash-to-virustotal.yaml diff --git a/workflows/security/enrichment/send-ip-to-virustotal.yaml b/examples/security/enrichment/send-ip-to-virustotal.yaml similarity index 100% rename from workflows/security/enrichment/send-ip-to-virustotal.yaml rename to examples/security/enrichment/send-ip-to-virustotal.yaml diff --git a/workflows/security/response/README.md b/examples/security/response/README.md similarity index 100% rename from workflows/security/response/README.md rename to examples/security/response/README.md diff --git a/workflows/security/response/ad-automated-triaging.yaml b/examples/security/response/ad-automated-triaging.yaml similarity index 100% rename from workflows/security/response/ad-automated-triaging.yaml rename to examples/security/response/ad-automated-triaging.yaml diff --git a/workflows/security/response/case-workflow-prod.yaml b/examples/security/response/case-workflow-prod.yaml similarity index 100% rename from workflows/security/response/case-workflow-prod.yaml rename to examples/security/response/case-workflow-prod.yaml diff --git a/workflows/security/response/createcasetool.yaml b/examples/security/response/createcasetool.yaml similarity index 100% rename from workflows/security/response/createcasetool.yaml rename to examples/security/response/createcasetool.yaml diff --git a/workflows/security/response/traditional-triage.yaml b/examples/security/response/traditional-triage.yaml similarity index 100% rename from workflows/security/response/traditional-triage.yaml rename to examples/security/response/traditional-triage.yaml diff --git a/workflows/utilities/README.md b/examples/utilities/README.md similarity index 100% rename from workflows/utilities/README.md rename to examples/utilities/README.md diff --git a/workflows/utilities/digest-workflow.yaml b/examples/utilities/digest-workflow.yaml similarity index 100% rename from workflows/utilities/digest-workflow.yaml rename to examples/utilities/digest-workflow.yaml diff --git a/workflows/utilities/execute-and-retrieve.yaml b/examples/utilities/execute-and-retrieve.yaml similarity index 100% rename from workflows/utilities/execute-and-retrieve.yaml rename to examples/utilities/execute-and-retrieve.yaml diff --git a/workflows/utilities/execute.yaml b/examples/utilities/execute.yaml similarity index 100% rename from workflows/utilities/execute.yaml rename to examples/utilities/execute.yaml diff --git a/workflows/utilities/get-action-status.yaml b/examples/utilities/get-action-status.yaml similarity index 100% rename from workflows/utilities/get-action-status.yaml rename to examples/utilities/get-action-status.yaml diff --git a/workflows/utilities/get-time.yaml b/examples/utilities/get-time.yaml similarity index 100% rename from workflows/utilities/get-time.yaml rename to examples/utilities/get-time.yaml diff --git a/workflows/utilities/index-commits.yaml b/examples/utilities/index-commits.yaml similarity index 100% rename from workflows/utilities/index-commits.yaml rename to examples/utilities/index-commits.yaml diff --git a/workflows/utilities/invoke-other-workflows.yaml b/examples/utilities/invoke-other-workflows.yaml similarity index 100% rename from workflows/utilities/invoke-other-workflows.yaml rename to examples/utilities/invoke-other-workflows.yaml diff --git a/workflows/utilities/save-emulation-results.yaml b/examples/utilities/save-emulation-results.yaml similarity index 100% rename from workflows/utilities/save-emulation-results.yaml rename to examples/utilities/save-emulation-results.yaml diff --git a/workflows/utilities/summarize-hackernews.yaml b/examples/utilities/summarize-hackernews.yaml similarity index 100% rename from workflows/utilities/summarize-hackernews.yaml rename to examples/utilities/summarize-hackernews.yaml diff --git a/workflows/utilities/update-emulation-results.yaml b/examples/utilities/update-emulation-results.yaml similarity index 100% rename from workflows/utilities/update-emulation-results.yaml rename to examples/utilities/update-emulation-results.yaml diff --git a/workflows/utilities/url-threat-scan.yaml b/examples/utilities/url-threat-scan.yaml similarity index 100% rename from workflows/utilities/url-threat-scan.yaml rename to examples/utilities/url-threat-scan.yaml diff --git a/kibana-versions.json b/kibana-versions.json new file mode 100644 index 0000000..e486333 --- /dev/null +++ b/kibana-versions.json @@ -0,0 +1,5 @@ +{ + "latest": "main", + "oldest": "9.5.0", + "cataloguePer": "minor" +} diff --git a/library/workflows/root-cause-analysis/root-cause-analysis.yaml b/library/workflows/root-cause-analysis/root-cause-analysis.yaml index a6a18d4..a29b2e6 100644 --- a/library/workflows/root-cause-analysis/root-cause-analysis.yaml +++ b/library/workflows/root-cause-analysis/root-cause-analysis.yaml @@ -8,6 +8,7 @@ template-metadata: possible root causes, then file a Case with the agent's analysis, a generated title and description, the alert attached, and a structured summary of the agent's reasoning steps as comments. + solutions: [security, observability] categories: [root-cause-analysis, ai-agent, case-management] install: form: diff --git a/library/workflows/semantic-knowledge-search/semantic-knowledge-search.yaml b/library/workflows/semantic-knowledge-search/semantic-knowledge-search.yaml index 9be2958..34514da 100644 --- a/library/workflows/semantic-knowledge-search/semantic-knowledge-search.yaml +++ b/library/workflows/semantic-knowledge-search/semantic-knowledge-search.yaml @@ -7,7 +7,6 @@ template-metadata: Run a semantic search against a knowledge-base index using a natural language query, with optional namespace and knowledge-type filters applied as Elasticsearch term clauses. - solutions: [search] categories: [search] install: form: diff --git a/library/workflows/web-search/web-search.yaml b/library/workflows/web-search/web-search.yaml index 1cbc108..e0cb1df 100644 --- a/library/workflows/web-search/web-search.yaml +++ b/library/workflows/web-search/web-search.yaml @@ -7,6 +7,7 @@ template-metadata: Run a web search via the Brave Search API and return the raw result set. Useful as a building block for enrichment, research, or LLM-grounding workflows. + solutions: [] categories: [search] icon: brave install: diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..dd95983 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,57 @@ +{ + "name": "@elastic/workflows-library", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@elastic/workflows-library", + "version": "0.0.0", + "license": "Apache-2.0", + "dependencies": { + "js-yaml": "^4.2.0", + "semver": "^7.8.4" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, + "node_modules/js-yaml": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.2.0.tgz", + "integrity": "sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/semver": { + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.4.tgz", + "integrity": "sha512-rUCObTnP32Q08R2uuIrt7r9PlEonuTmtuXYcW6s5kjdlj3xbnwe+21yXptAUYcMAABLkYYTtnmzb3w3EDZfueA==", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..502ef93 --- /dev/null +++ b/package.json @@ -0,0 +1,18 @@ +{ + "name": "@elastic/workflows-library", + "version": "0.0.0", + "private": true, + "description": "Source repo for the Kibana Workflow Template Library", + "license": "Apache-2.0", + "type": "module", + "engines": { + "node": ">=20" + }, + "scripts": { + "build:catalog": "node scripts/build-catalog.mjs" + }, + "dependencies": { + "js-yaml": "^4.2.0", + "semver": "^7.8.4" + } +} diff --git a/scripts/build-catalog.mjs b/scripts/build-catalog.mjs new file mode 100644 index 0000000..ae35696 --- /dev/null +++ b/scripts/build-catalog.mjs @@ -0,0 +1,433 @@ +#!/usr/bin/env node +/** + * build-catalog.mjs — Workflow Template Library catalog generator. + * + * Reads: + * - kibana-versions.json (policy: latest channel, oldest supported minor, + * catalogue granularity) + * - library/categories.yaml (closed-vocab category registry) + * - library/workflows//.yaml (templates) + * + * Writes (under dist/v1/): + * - kibana-versions.json (resolved, consumer-facing) + * - /catalogs/templates.json (one per active Kibana version) + * - /manifest.json (with effectiveKibanaSemver) + * - templates//.yaml (raw template body, version-keyed) + * + * Resolves the list of Kibana versions to catalogue dynamically: + * - `main`'s semver is fetched from elastic/kibana@main's package.json + * (overridable via KIBANA_MAIN_VERSION env var for local dev / recovery). + * - Named minors are discovered from elastic/kibana's branch list via the + * GitHub API, filtered by `oldest` (semver floor) and `cataloguePer`. + * + * Fails closed: a malformed template, an unknown category, an unreachable + * Kibana main, or a missing required field aborts the publish — never falls + * back to stale data. + */ + +import { readFile, writeFile, mkdir, readdir } from 'node:fs/promises'; +import { existsSync } from 'node:fs'; +import { createHash } from 'node:crypto'; +import path from 'node:path'; +import yaml from 'js-yaml'; +import semver from 'semver'; + +// --- Paths --------------------------------------------------------------- + +const REPO_ROOT = process.cwd(); +const POLICY_FILE = path.join(REPO_ROOT, 'kibana-versions.json'); +const CATEGORIES_FILE = path.join(REPO_ROOT, 'library/categories.yaml'); +const TEMPLATES_DIR = path.join(REPO_ROOT, 'library/workflows'); +const OUT = path.join(REPO_ROOT, 'dist/v1'); + +// --- Helpers ------------------------------------------------------------- + +const log = (...args) => console.log('[build-catalog]', ...args); + +function readJson(file) { + return readFile(file, 'utf8').then(JSON.parse); +} + +function readYaml(file) { + return readFile(file, 'utf8').then((raw) => yaml.load(raw)); +} + +function sha256(buf) { + return 'sha256:' + createHash('sha256').update(buf).digest('hex'); +} + +// --- Step 1: Load policy file -------------------------------------------- + +async function loadPolicy() { + const policy = await readJson(POLICY_FILE); + + if (!policy.latest) throw new Error('kibana-versions.json: missing `latest`'); + if (!policy.oldest) throw new Error('kibana-versions.json: missing `oldest`'); + if (!policy.cataloguePer) throw new Error('kibana-versions.json: missing `cataloguePer`'); + + if (!semver.valid(policy.oldest)) { + throw new Error(`kibana-versions.json: \`oldest\` must be a valid semver, got '${policy.oldest}'`); + } + if (policy.cataloguePer !== 'minor') { + throw new Error(`kibana-versions.json: \`cataloguePer\` only supports 'minor' for now, got '${policy.cataloguePer}'`); + } + + return policy; +} + +// --- Step 2: Resolve `main`'s semver ------------------------------------- + +async function resolveMainKibanaSemver() { + const override = process.env.KIBANA_MAIN_VERSION; + if (override) { + if (!semver.valid(override)) { + throw new Error(`KIBANA_MAIN_VERSION env var is not valid semver: '${override}'`); + } + log(`Using KIBANA_MAIN_VERSION override: ${override}`); + return override; + } + + const url = 'https://raw.githubusercontent.com/elastic/kibana/main/package.json'; + log(`Resolving main semver from ${url}`); + const res = await fetch(url); + if (!res.ok) { + throw new Error(`Failed to fetch ${url}: HTTP ${res.status}. Cannot resolve main's Kibana version.`); + } + const pkg = await res.json(); + if (!pkg.version) { + throw new Error(`Kibana main package.json has no \`version\` field.`); + } + // Strip any pre-release suffix (`-snapshot`, `-pre`, etc.) before validating. + const clean = semver.coerce(pkg.version)?.version; + if (!clean || !semver.valid(clean)) { + throw new Error(`Kibana main package.json .version='${pkg.version}' did not normalize to a valid semver`); + } + return clean; +} + +// --- Step 3: Discover supported named minors from Kibana branches -------- + +/** + * Builds a multi-line, actionable error message for a non-2xx response from + * the GitHub API. Distinguishes rate-limit 403s from auth 401/403s and points + * the operator at the two escape hatches (GITHUB_TOKEN, KIBANA_NAMED_MINORS). + */ +function formatGitHubApiError(res, url, hasToken) { + const remaining = res.headers.get('x-ratelimit-remaining'); + const reset = res.headers.get('x-ratelimit-reset'); + const limit = res.headers.get('x-ratelimit-limit'); + + const lines = [`Failed to list elastic/kibana branches: HTTP ${res.status} from ${url}`]; + + const isRateLimit = (res.status === 403 || res.status === 429) && remaining === '0'; + + if (isRateLimit) { + const resetAt = reset ? new Date(Number(reset) * 1000).toISOString() : 'unknown'; + lines.push(''); + lines.push(`Cause: GitHub API rate limit exhausted (used ${limit}/${limit}, resets at ${resetAt}).`); + if (!hasToken) { + lines.push(''); + lines.push('You are calling the GitHub API unauthenticated, which is rate-limited to 60 requests/hour per IP.'); + lines.push(''); + lines.push('Fixes (pick one):'); + lines.push(' 1. Authenticate the call (raises the limit to 5,000/hour):'); + lines.push(' export GITHUB_TOKEN=$(gh auth token) # if you use the gh CLI'); + lines.push(' # or any classic/fine-grained PAT — no scopes needed for public repos'); + lines.push(' npm run build:catalog'); + lines.push(' 2. Skip the branch fetch entirely (for quick local iteration):'); + lines.push(' KIBANA_NAMED_MINORS="" npm run build:catalog # zero named minors'); + lines.push(' KIBANA_NAMED_MINORS="9.5,9.6" npm run build:catalog # specific minors'); + lines.push(' 3. Wait until the rate limit window resets and retry.'); + } else { + lines.push(''); + lines.push('Your GITHUB_TOKEN is set but the authenticated rate limit (5,000/hour) is also exhausted.'); + lines.push('Wait until the reset time above, or use KIBANA_NAMED_MINORS to skip the API entirely.'); + } + } else if (res.status === 401 || res.status === 403) { + lines.push(''); + if (hasToken) { + lines.push('Cause: GITHUB_TOKEN is set but was rejected by GitHub (expired, revoked, or invalid).'); + lines.push(''); + lines.push('Fixes:'); + lines.push(' 1. Refresh your token:'); + lines.push(' export GITHUB_TOKEN=$(gh auth token)'); + lines.push(' 2. Or unset it to fall back to unauthenticated access (60 req/h is plenty for one run):'); + lines.push(' unset GITHUB_TOKEN && npm run build:catalog'); + lines.push(' 3. Or skip the branch fetch entirely:'); + lines.push(' KIBANA_NAMED_MINORS="" npm run build:catalog'); + } else { + lines.push('Cause: GitHub returned 403 without a rate-limit signature. Possibly an IP-level block or a transient issue.'); + lines.push(''); + lines.push('Try again, or skip the API:'); + lines.push(' KIBANA_NAMED_MINORS="" npm run build:catalog'); + } + } else { + lines.push(''); + lines.push('Unexpected HTTP status from the GitHub API. Re-run; if it persists, skip the API:'); + lines.push(' KIBANA_NAMED_MINORS="" npm run build:catalog'); + } + + return lines.join('\n'); +} + +async function discoverNamedMinors(oldest) { + // Local-dev / recovery escape hatch: skip the GitHub API entirely. + // KIBANA_NAMED_MINORS="" → treat as zero named minors + // KIBANA_NAMED_MINORS="9.5,9.6" → use those exact minors + const override = process.env.KIBANA_NAMED_MINORS; + if (override !== undefined) { + const items = override + .split(',') + .map((s) => s.trim()) + .filter(Boolean) + .map((id) => { + if (!/^\d+\.\d+$/.test(id)) { + throw new Error(`KIBANA_NAMED_MINORS contains non-minor id '${id}' (expected '.')`); + } + return { id, kibana: `${id}.0`, active: true }; + }); + log(`Using KIBANA_NAMED_MINORS override: [${items.map((i) => i.id).join(', ') || '(empty)'}]`); + return items; + } + + const url = 'https://api.github.com/repos/elastic/kibana/branches?per_page=100'; + const hasToken = Boolean(process.env.GITHUB_TOKEN); + log(`Discovering named minors from ${url} (${hasToken ? 'authenticated' : 'unauthenticated'})`); + + // GitHub branches API paginates. Walk pages until empty. + let page = 1; + const branchNames = []; + while (true) { + const headers = { Accept: 'application/vnd.github+json' }; + if (hasToken) { + headers.Authorization = `Bearer ${process.env.GITHUB_TOKEN}`; + } + const res = await fetch(`${url}&page=${page}`, { headers }); + if (!res.ok) { + throw new Error(formatGitHubApiError(res, url, hasToken)); + } + const batch = await res.json(); + if (!batch.length) break; + branchNames.push(...batch.map((b) => b.name)); + if (batch.length < 100) break; + page++; + } + + // Keep only branches that match `^.$`. Filter by `oldest`. + const oldestParsed = semver.parse(oldest); + const minors = branchNames + .map((name) => { + const m = /^(\d+)\.(\d+)$/.exec(name); + if (!m) return null; + const major = Number(m[1]); + const minor = Number(m[2]); + const repSemver = `${major}.${minor}.0`; + return { id: name, kibana: repSemver, major, minor }; + }) + .filter(Boolean) + .filter((b) => semver.gte(b.kibana, `${oldestParsed.major}.${oldestParsed.minor}.0`)) + // Highest first, deterministic order. + .sort((a, b) => semver.rcompare(a.kibana, b.kibana)) + .map(({ id, kibana }) => ({ id, kibana, active: true })); + + log(`Discovered ${minors.length} named minor(s): ${minors.map((m) => m.id).join(', ') || '(none)'}`); + return minors; +} + +// --- Step 4: Load categories vocabulary ---------------------------------- + +async function loadCategoriesVocab() { + const vocab = await readYaml(CATEGORIES_FILE); + if (!vocab?.categories?.length) { + throw new Error('library/categories.yaml: missing or empty `categories` array'); + } + return new Set(vocab.categories.map((c) => c.id)); +} + +// --- Step 5: Discover and validate every template ------------------------ + +async function loadTemplates(validCategoryIds) { + const slugs = await readdir(TEMPLATES_DIR); + const templates = []; + + for (const slug of slugs.sort()) { + const file = path.join(TEMPLATES_DIR, slug, `${slug}.yaml`); + if (!existsSync(file)) continue; + const raw = await readFile(file, 'utf8'); + const parsed = yaml.load(raw); + const meta = parsed?.['template-metadata']; + + if (!meta) { + throw new Error(`${file}: missing \`template-metadata\` block`); + } + for (const required of ['slug', 'version', 'availability', 'name', 'description', 'categories']) { + if (meta[required] === undefined) { + throw new Error(`${file}: missing required field \`template-metadata.${required}\``); + } + } + // `solutions` is optional. Absent → cross-solution template (available everywhere). + if (meta.solutions !== undefined) { + if (!Array.isArray(meta.solutions) || meta.solutions.length === 0) { + meta.solutions = undefined; // auto-set to cross-solution template + } + } + if (meta.slug !== slug) { + throw new Error(`${file}: slug '${meta.slug}' does not match parent directory '${slug}'`); + } + if (!semver.valid(meta.version)) { + throw new Error(`${file}: \`version\` must be a valid semver, got '${meta.version}'`); + } + if (!semver.validRange(meta.availability)) { + throw new Error(`${file}: \`availability\` must be a valid semver range, got '${meta.availability}'`); + } + const unknownCats = meta.categories.filter((c) => !validCategoryIds.has(c)); + if (unknownCats.length) { + throw new Error( + `${file}: unknown categories [${unknownCats.join(', ')}]. Add them to library/categories.yaml in the same PR.` + ); + } + // Body-level cross-check: every `__install__.` reference must have a matching install.form entry. + const formFieldNames = new Set((meta.install?.form ?? []).map((f) => f.name)); + const bodyOnly = { ...parsed }; + delete bodyOnly['template-metadata']; + const bodyStr = yaml.dump(bodyOnly); + const installRefs = new Set([...bodyStr.matchAll(/__install__\.([a-zA-Z0-9_-]+)/g)].map((m) => m[1])); + const missingForm = [...installRefs].filter((r) => !formFieldNames.has(r)); + if (missingForm.length) { + throw new Error( + `${file}: \`__install__.{${missingForm.join(', ')}}\` referenced but not declared in install.form` + ); + } + + templates.push({ + slug: meta.slug, + version: meta.version, + availability: meta.availability, + metadata: meta, + body: raw, + contentHash: sha256(raw), + fixedConnectors: deriveFixedConnectors(parsed), + }); + } + + if (!templates.length) { + throw new Error(`No templates discovered under ${TEMPLATES_DIR}`); + } + log(`Loaded ${templates.length} template(s)`); + return templates; +} + +// `fixedConnectors` lists the connector-type IDs the template hard-codes via +// step `type:` (the part before the first `.`). Templates that route all +// connector binding through `__install__.` have an empty array. +function deriveFixedConnectors(parsed) { + const steps = parsed?.steps ?? []; + const types = new Set(); + for (const step of steps) { + if (typeof step?.type !== 'string') continue; + const dot = step.type.indexOf('.'); + if (dot <= 0) continue; + const prefix = step.type.slice(0, dot); + // Skip in-house Kibana / engine namespaces — only surface external connectors. + if (['console', 'http', 'data', 'workflow', 'kibana', 'cases', 'elasticsearch', 'security', 'ai', 'inference', 'foreach'].includes(prefix)) continue; + types.add(prefix); + } + return [...types].sort(); +} + +// --- Step 6: Build the catalog ------------------------------------------- + +function pickTemplatesFor(kibanaSemver, allTemplates) { + const bySlug = new Map(); + for (const t of allTemplates) { + if (!semver.satisfies(kibanaSemver, t.availability)) continue; + const prev = bySlug.get(t.slug); + if (!prev || semver.gt(t.version, prev.version)) bySlug.set(t.slug, t); + } + return [...bySlug.values()].sort((a, b) => a.slug.localeCompare(b.slug)); +} + +function templateRow(t) { + return { + slug: t.slug, + version: t.version, + availability: t.availability, + name: t.metadata.name, + description: t.metadata.description, + solutions: t.metadata.solutions, + categories: t.metadata.categories, + icon: t.metadata.icon, + definitionUrl: `templates/${t.slug}/${t.version}.yaml`, + contentHash: t.contentHash, + fixedConnectors: t.fixedConnectors, + }; +} + +// --- Main ---------------------------------------------------------------- + +async function main() { + const policy = await loadPolicy(); + const mainSemver = await resolveMainKibanaSemver(); + const namedMinors = await discoverNamedMinors(policy.oldest); + const validCategoryIds = await loadCategoriesVocab(); + const allTemplates = await loadTemplates(validCategoryIds); + + // Compose the resolved Kibana-versions list: every named minor + `main` sentinel. + const resolvedVersions = [ + ...namedMinors, + { id: 'main', kibana: mainSemver, active: true }, + ]; + log(`Resolved main → Kibana ${mainSemver}`); + + // Wipe + recreate dist/v1. + await mkdir(OUT, { recursive: true }); + + // Emit the resolved kibana-versions.json (consumer-facing shape). + await writeFile( + path.join(OUT, 'kibana-versions.json'), + JSON.stringify({ versions: resolvedVersions, latest: policy.latest }, null, 2) + '\n' + ); + + // Emit per-version templates.json + manifest.json. + const generatedAt = new Date().toISOString(); + for (const v of resolvedVersions) { + if (v.active === false) continue; + + const rows = pickTemplatesFor(v.kibana, allTemplates).map(templateRow); + const catalogsDir = path.join(OUT, v.id, 'catalogs'); + await mkdir(catalogsDir, { recursive: true }); + + const templatesJsonBody = + JSON.stringify({ version: 'v1', kibanaVersion: v.kibana, generatedAt, templates: rows }, null, 2) + '\n'; + await writeFile(path.join(catalogsDir, 'templates.json'), templatesJsonBody); + + const manifest = { + version: 'v1', + kibanaVersionId: v.id, + effectiveKibanaSemver: v.kibana, + generatedAt, + hashes: { 'catalogs/templates.json': sha256(templatesJsonBody) }, + }; + await writeFile( + path.join(OUT, v.id, 'manifest.json'), + JSON.stringify(manifest, null, 2) + '\n' + ); + + log(` → v1/${v.id}/ (kibana ${v.kibana}, ${rows.length} template${rows.length === 1 ? '' : 's'})`); + } + + // Emit each template body once at its version-keyed URL. + for (const t of allTemplates) { + const dir = path.join(OUT, 'templates', t.slug); + await mkdir(dir, { recursive: true }); + await writeFile(path.join(dir, `${t.version}.yaml`), t.body); + } + log(` → v1/templates/ (${allTemplates.length} version-keyed YAML bodies)`); + + log('Done.'); +} + +main().catch((err) => { + console.error('[build-catalog] FAILED:', err.message); + process.exit(1); +});