From 9831e8c835b1df627f95cf7db7538fce6ed24230 Mon Sep 17 00:00:00 2001 From: Yvonne Date: Wed, 27 May 2026 09:39:11 -0400 Subject: [PATCH] =?UTF-8?q?docs:=20restructure=20/sync=20=E2=80=94=20uv-st?= =?UTF-8?q?yle=20sidebar,=207=20new=20pages,=20voice=20pass?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - New pages: orchestration (schedule sync runs), side-by-side pattern, migration walkthrough, schema mapping reference, choosing an adapter, PeeringDB + Device42 adapter stubs. - Moved: guides/ contents (installation, creation→create, run, custom-certificates) to docs/docs/ root. - Split: development.mdx → contributing.mdx (contributor docs) + RELEASING.md (maintainer release runbook) at repo root. - Sidebar: uv-style structure — Get started · Guides · Adapters · Reference · Contributing. - Titles: imperative form for task pages ("Create a sync project"), noun form for reference. - Voice pass on all new prose: drop "the team" third-person paraphrasing, "without X" negation framing, and corporate-speak verb+adverb constructions; lead with capability + use case, not implementation type. - 5 URL redirects via @docusaurus/plugin-client-redirects (legacy /guides/* paths + /development → /contributing). Co-Authored-By: Claude Opus 4.7 (1M context) --- .vale/styles/spelling-exceptions.txt | 10 + README.md | 12 +- docs/docs/development.mdx => RELEASING.md | 190 ++--------- docs/docs/adapters/choosing-an-adapter.mdx | 78 +++++ docs/docs/adapters/device42.mdx | 24 ++ docs/docs/adapters/nautobot.mdx | 7 + docs/docs/adapters/netbox.mdx | 7 + docs/docs/adapters/peeringdb.mdx | 24 ++ docs/docs/contributing.mdx | 147 +++++++++ docs/docs/creating-a-sync-project.mdx | 109 ++++++ docs/docs/custom-certificates.mdx | 32 ++ docs/docs/guides/creation.mdx | 112 ------- docs/docs/guides/custom-certificates.mdx | 29 -- docs/docs/{guides => }/installation.mdx | 2 +- .../migrating-from-netbox-or-nautobot.mdx | 143 ++++++++ docs/docs/orchestration.mdx | 96 ++++++ docs/docs/readme.mdx | 26 +- docs/docs/reference/config.mdx | 23 +- docs/docs/reference/schema-mapping.mdx | 310 ++++++++++++++++++ .../{guides/run.mdx => running-a-sync.mdx} | 10 +- ...using-netbox-or-nautobot-with-infrahub.mdx | 88 +++++ docs/docusaurus.config.ts | 30 ++ docs/package-lock.json | 25 ++ docs/package.json | 1 + docs/sidebars.ts | 22 +- 25 files changed, 1198 insertions(+), 359 deletions(-) rename docs/docs/development.mdx => RELEASING.md (58%) create mode 100644 docs/docs/adapters/choosing-an-adapter.mdx create mode 100644 docs/docs/adapters/device42.mdx create mode 100644 docs/docs/adapters/peeringdb.mdx create mode 100644 docs/docs/contributing.mdx create mode 100644 docs/docs/creating-a-sync-project.mdx create mode 100644 docs/docs/custom-certificates.mdx delete mode 100644 docs/docs/guides/creation.mdx delete mode 100644 docs/docs/guides/custom-certificates.mdx rename docs/docs/{guides => }/installation.mdx (90%) create mode 100644 docs/docs/migrating-from-netbox-or-nautobot.mdx create mode 100644 docs/docs/orchestration.mdx create mode 100644 docs/docs/reference/schema-mapping.mdx rename docs/docs/{guides/run.mdx => running-a-sync.mdx} (89%) create mode 100644 docs/docs/using-netbox-or-nautobot-with-infrahub.mdx diff --git a/.vale/styles/spelling-exceptions.txt b/.vale/styles/spelling-exceptions.txt index ed00e31..7fd0de5 100644 --- a/.vale/styles/spelling-exceptions.txt +++ b/.vale/styles/spelling-exceptions.txt @@ -88,6 +88,7 @@ modularization namespace namespaces Nautobot +Nautobot's Netbox Newsfragment Nornir @@ -131,6 +132,15 @@ uv validator validators Version Control +VRFs +walkthrough +runbook +runbooks +Splunk +Datadog +Infoblox +PeeringDB +Device42 Vitest VLANs vscode diff --git a/README.md b/README.md index a7cc0c8..a1c9f73 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ pip install infrahub-sync infrahub-sync --help ``` -→ For step-by-step setup, see [Installing Infrahub Sync](https://docs.infrahub.app/sync/guides/installation), [Creating a new sync instance](https://docs.infrahub.app/sync/guides/creation), and [Running sync tasks](https://docs.infrahub.app/sync/guides/run). +→ For step-by-step setup, see [Install Infrahub Sync](https://docs.infrahub.app/sync/installation), [Create a sync project](https://docs.infrahub.app/sync/creating-a-sync-project), and [Run a sync](https://docs.infrahub.app/sync/running-a-sync). --- @@ -63,7 +63,7 @@ infrahub-sync sync --name from-netbox --directory ./examples After the sync completes, every NetBox device, interface, VLAN, prefix, and related model appears in Infrahub, mapped per the schema mapping in `config.yml`. -→ For the full walkthrough of what happens under the hood — adapter generation, diff calculation, and sync ordering — see [Running sync tasks](https://docs.infrahub.app/sync/guides/run). +→ For the full walkthrough of what happens under the hood — adapter generation, diff calculation, and sync ordering — see [Run a sync](https://docs.infrahub.app/sync/running-a-sync). --- @@ -113,13 +113,13 @@ After the sync completes, every NetBox device, interface, VLAN, prefix, and rela | | | |---|---| -| **Install and run** | [Installing Infrahub Sync](https://docs.infrahub.app/sync/guides/installation) · [Creating a sync instance](https://docs.infrahub.app/sync/guides/creation) · [Running sync tasks](https://docs.infrahub.app/sync/guides/run) | +| **Install and run** | [Install Infrahub Sync](https://docs.infrahub.app/sync/installation) · [Create a sync project](https://docs.infrahub.app/sync/creating-a-sync-project) · [Run a sync](https://docs.infrahub.app/sync/running-a-sync) | | **Full documentation** | [Infrahub Sync docs](https://docs.infrahub.app/sync) | | **All adapters** | [Adapter reference](https://docs.infrahub.app/sync#adapters) | | **Configuration reference** | [Sync instance configuration](https://docs.infrahub.app/sync/reference/config) · [CLI reference](https://docs.infrahub.app/sync/reference/cli) | -| **Custom CA certificates** | [Custom certificates guide](https://docs.infrahub.app/sync/guides/custom-certificates) | +| **Custom CA certificates** | [Custom certificates guide](https://docs.infrahub.app/sync/custom-certificates) | | **Local custom adapters** | [Local adapters guide](https://docs.infrahub.app/sync/adapters/local-adapters) | -| **Contribute** | [Development guide](https://docs.infrahub.app/sync/development) — development environment, tests, releases | +| **Contribute** | [Contributing guide](https://docs.infrahub.app/sync/contributing) — development environment, tests, code standards | --- @@ -127,7 +127,7 @@ After the sync completes, every NetBox device, interface, VLAN, prefix, and rela - **Report a bug or request a feature** — [GitHub Issues](https://github.com/opsmill/infrahub-sync/issues) - **Discuss with the community** — [discord.gg/opsmill](https://discord.gg/opsmill) -- **Contribute code or docs** — see the [Development guide](https://docs.infrahub.app/sync/development) +- **Contribute code or docs** — see the [Contributing guide](https://docs.infrahub.app/sync/contributing) --- diff --git a/docs/docs/development.mdx b/RELEASING.md similarity index 58% rename from docs/docs/development.mdx rename to RELEASING.md index 86ec275..9214add 100644 --- a/docs/docs/development.mdx +++ b/RELEASING.md @@ -1,156 +1,8 @@ ---- -title: Development ---- +# Releasing Infrahub Sync -This guide covers how to set up a development environment for `infrahub-sync`, contribute to the project, and publish releases. +Maintainer-only runbook for publishing new releases of `infrahub-sync` to [PyPI](https://pypi.org/project/infrahub-sync/). -## Prerequisites - -- Python 3.10–3.13 (3.12 recommended) -- [uv](https://docs.astral.sh/uv/) for dependency management -- Git - -## Setting up your development environment - -### Clone the repository - -```bash -git clone https://github.com/opsmill/infrahub-sync.git -cd infrahub-sync -``` - -### Install uv - -If you don't have uv installed, you can install it with: - -```bash -curl -LsSf https://astral.sh/uv/install.sh | sh -``` - -Or see the [uv installation guide](https://docs.astral.sh/uv/getting-started/installation/) for other options. - -### Install dependencies - -```bash -uv sync --group dev -``` - -This installs all runtime and development dependencies defined in `pyproject.toml`. - -### Verify your setup - -```bash -uv run infrahub-sync --help -uv run infrahub-sync list --directory examples/ -``` - -## Development workflow - -Before committing any changes, run the following commands in order: - -```bash -uv run invoke format # Format code with ruff -uv run invoke lint # Lint code with ruff and pylint -uv run mypy infrahub_sync/ --ignore-missing-imports -``` - -### Validate the CLI - -After making changes, verify the CLI still works: - -```bash -uv run infrahub-sync --help -uv run infrahub-sync list --directory examples/ -uv run infrahub-sync generate --name from-netbox --directory examples/ -``` - -### Running tests - -```bash -uv run pytest -q -``` - -## Code standards - -### Python style - -- Python 3.10–3.13 compatible -- Type hints on new or changed code -- Ruff-formatted and lint-clean -- Mypy-checked (do not increase existing error count) -- Public functions and classes require documentation strings -- Raise specific exceptions; avoid broad `except Exception:` - -### Line length - -- Maximum line length: 120 characters (configured in `pyproject.toml`) - -## Documentation - -If you make user-facing changes (CLI flags, configuration options, new adapters), update the documentation. - -### Generate command-line documentation - -```bash -uv run invoke docs.generate -``` - -### Build documentation site - -First-time setup (requires Node.js): - -```bash -cd docs && npm install -``` - -Build the site: - -```bash -uv run invoke docs.docusaurus -``` - -### Lint markdown files - -```bash -npx markdownlint-cli "docs/docs/**/*.{md,mdx}" -npx markdownlint-cli --fix "docs/docs/**/*.{md,mdx}" -``` - -## Adding a new adapter - -1. Create `infrahub_sync/adapters/.py` following existing adapter patterns -2. Add connection configuration schema and an example under `examples/` -3. Provide `list` and `diff` pathways before enabling `sync` -4. Document required environment variables and expected error cases -5. Create a documentation page in `docs/docs/adapters/` -6. Add the adapter to the sidebar in `docs/sidebars.ts` - -## Invoke tasks - -View all available tasks: - -```bash -uv run invoke --list -``` - -Common tasks: - -| Task | Description | -|------|-------------| -| `linter.format-ruff` | Format Python code with ruff | -| `linter.lint-ruff` | Lint Python code with ruff | -| `linter.lint-pylint` | Lint Python code with pylint | -| `linter.lint-yaml` | Lint YAML files with yamllint | -| `docs.generate` | Generate CLI documentation | -| `docs.docusaurus` | Build documentation website | -| `format` | Alias for ruff format | -| `lint` | Run all linters | - -## Publishing a release - -This section documents how to publish new releases of `infrahub-sync` to PyPI. - -### Overview +## Overview The project uses an automated release system powered by GitHub Actions. There are three ways to publish a release: @@ -158,7 +10,7 @@ The project uses an automated release system powered by GitHub Actions. There ar 2. **Manual GitHub release** (for controlled releases) 3. **Manual workflow dispatch** (for emergency or custom releases) -### Prerequisites +## Prerequisites Before publishing, ensure: @@ -166,11 +18,11 @@ Before publishing, ensure: - The `PYPI_TOKEN` secret is configured in repository settings - The `GH_INFRAHUB_BOT_TOKEN` secret is configured (for automated releases) -### Method 1: Automated release (recommended) +## Method 1: Automated release (recommended) This is the standard release flow. Releases are triggered automatically when PRs are merged to `main` or `stable` branches. -#### Step 1: Label your pull requests +### Step 1: Label your pull requests Apply appropriate labels to PRs before merging. Labels determine the version bump: @@ -189,7 +41,7 @@ Auto-labeling rules are configured in `.github/release-drafter.yml` but require | Contains `chore` | `ci/skip-changelog` | | Contains `deprecat` | `type/deprecated` | -#### Step 2: Merge to main +### Step 2: Merge to main Merge your labeled PR to the `main` branch. The automation will: @@ -198,7 +50,7 @@ Merge your labeled PR to the `main` branch. The automation will: 3. Commit changes as `chore(release): v{VERSION} [skip ci]` 4. Create/update a draft GitHub Release with auto-generated release notes -#### Step 3: Publish the GitHub release +### Step 3: Publish the GitHub release 1. Navigate to the repository's **Releases** page 2. Find the draft release created by Release Drafter @@ -208,11 +60,11 @@ Merge your labeled PR to the `main` branch. The automation will: Publishing the release triggers the PyPI upload automatically. -### Method 2: Manual GitHub release +## Method 2: Manual GitHub release Use this method when you want full control over the release timing and notes. -#### Step 1: Update the version +### Step 1: Update the version Update the version in `pyproject.toml`: @@ -229,7 +81,7 @@ git commit -m "chore(release): vX.Y.Z" git push origin main ``` -#### Step 2: Create a GitHub release +### Step 2: Create a GitHub release 1. Go to **Releases** → **Draft a new release** 2. Click **Choose a tag** and create a new tag matching your version (for example, `1.6.0`) @@ -240,11 +92,11 @@ git push origin main This triggers the `trigger-release.yml` workflow, which publishes to PyPI. -### Method 3: Manual workflow dispatch +## Method 3: Manual workflow dispatch Use this for emergency releases or when you need to bypass the standard flow. -#### Using the GitHub UI +### Using the GitHub UI 1. Go to **Actions** → **Publish Infrahub Sync Package** 2. Click **Run workflow** @@ -254,7 +106,7 @@ Use this for emergency releases or when you need to bypass the standard flow. - `runs-on`: OS for the runner (default: `ubuntu-22.04`) 4. Click **Run workflow** -#### Using the GitHub CLI +### Using the GitHub CLI ```bash gh workflow run workflow-publish.yml \ @@ -264,7 +116,7 @@ gh workflow run workflow-publish.yml \ **Important:** When using workflow dispatch, ensure `pyproject.toml` already has the correct version, as this method builds from the current code state. -### Release notes +## Release notes Release notes are auto-generated based on merged PRs and their labels: @@ -280,7 +132,7 @@ PRs with these labels are excluded from release notes: - `ci/skip-changelog` - `type/duplicate` -### Verifying a release +## Verifying a release After publishing: @@ -293,9 +145,9 @@ pip install infrahub-sync== infrahub-sync --version ``` -### Troubleshooting +## Troubleshooting -#### Release workflow skipped +### Release workflow skipped The automated release is skipped when: @@ -303,7 +155,7 @@ The automated release is skipped when: - No version bump is detected (no labeled PRs since last release) - Changes are only in the `docs/` directory -#### The PyPI upload failed +### The PyPI upload failed Common causes: @@ -313,11 +165,11 @@ Common causes: To retry, use the manual workflow dispatch method. -#### Version not bumped correctly +### Version not bumped correctly Ensure PRs have appropriate labels before merging. If labels are missing, the version drafter may not calculate a new version. -### Workflow files reference +## Workflow files reference | Workflow | Type | Purpose | |----------|------|---------| diff --git a/docs/docs/adapters/choosing-an-adapter.mdx b/docs/docs/adapters/choosing-an-adapter.mdx new file mode 100644 index 0000000..1367ab6 --- /dev/null +++ b/docs/docs/adapters/choosing-an-adapter.mdx @@ -0,0 +1,78 @@ +--- +title: Choose an adapter +--- + +Infrahub Sync includes adapters for the most common infrastructure systems and a Generic REST API adapter for everything else. For systems with non-standard APIs, build a custom adapter. + +## How to choose + +If a pre-built adapter exists for the system you want to connect, use it. If the system has a REST/JSON API but no dedicated adapter, use the Generic REST API adapter. Only build a custom adapter when the system has a non-standard API, requires custom authentication logic, or has data access patterns the Generic REST API adapter cannot handle. + +## Pre-built adapters + +Each adapter handles two responsibilities for its target system: **communication** (talking to the system's API, authentication, request handling) and **translation** (converting the system's data structures into the `diffsync` models the sync engine works with). + +| Adapter | Direction supported | Best for | +| --- | --- | --- | +| Infrahub | source or destination | Any sync project where Infrahub is one side of the sync | +| NetBox | NetBox → Infrahub | Existing NetBox deployments migrating to or syncing with Infrahub | +| Nautobot | Nautobot → Infrahub | Existing Nautobot deployments migrating to or syncing with Infrahub | +| IP Fabric | IP Fabric → Infrahub | Building Infrahub inventory from IP Fabric network discovery data | +| Slurp'it | Slurp'it → Infrahub | Building Infrahub inventory from Slurp'it network discovery data | +| Cisco ACI | Cisco ACI → Infrahub | Syncing data from Cisco ACI fabrics into Infrahub | +| Peering Manager | Peering Manager → Infrahub · Infrahub → Peering Manager | Bi-directional sync between Peering Manager and Infrahub | +| PeeringDB | PeeringDB → Infrahub | Pulling autonomous system and internet exchange data from PeeringDB into Infrahub | +| Prometheus | Prometheus → Infrahub | Pulling target or metric metadata from Prometheus into Infrahub | +| LibreNMS | LibreNMS → Infrahub | Syncing device inventory and metadata from LibreNMS into Infrahub | +| Observium | Observium → Infrahub | Syncing device inventory and metadata from Observium into Infrahub | +| Device42 | Device42 → Infrahub | Syncing IT asset and dependency data from Device42 into Infrahub | +| Generic REST API | external system → Infrahub | Any external system with a REST/JSON API and no dedicated adapter | + +For per-adapter details — connection parameters, supported models, known limitations — see the individual adapter pages in the Adapters sidebar. + +## When to use the Generic REST API adapter + +The Generic REST API adapter works with any system that exposes data over HTTP/JSON. It handles authentication and basic data access patterns out of configuration alone. Set the API endpoint, the authentication method, and the structure of the data you want to read in the same YAML format as every other adapter. + +Common cases where the Generic REST API adapter is the right choice: + +- **ServiceNow** — pull device, location, or service data from a ServiceNow CMDB. +- **Infoblox** — pull IP address management data from Infoblox. +- **Internal IPAM tools** — pull data from a homegrown IPAM or asset database. +- **Vendor-specific inventory APIs** — pull data from a vendor's cloud-hosted inventory service. +- **Spreadsheet-based data** — when a team exports a spreadsheet to a JSON file served over HTTP, the Generic REST API adapter can read it. + +Most source systems without a dedicated adapter are reachable directly with the Generic REST API adapter. See [Generic REST API adapter](./genericrestapi.mdx) for configuration details. + +## When to build a custom adapter + +A custom adapter is the right choice when the Generic REST API adapter can't handle the source system. Common reasons: + +- **Non-REST APIs.** The system uses gRPC, SOAP, GraphQL with non-standard patterns, a binary protocol, or any other non-REST interface. +- **Custom authentication flows.** The system requires multi-step authentication, token refresh logic, or other auth patterns standard configuration cannot express. +- **Stateful or session-based access.** The system requires maintaining a session across multiple requests in a way the Generic REST API adapter does not support. +- **Complex pagination or query patterns.** The data access requires logic beyond what configuration can express — for example, conditional pagination based on response content. +- **Data transformations that exceed Jinja filter capabilities.** The source data requires substantial preprocessing before mapping into `diffsync` models. + +A custom adapter is a Python class implementing the same interface as the pre-built adapters. Load it from a filesystem path, a Python module path, or an installed entry point. See [Local adapters](./local-adapters.mdx) for how to build and load one. + +If you build a custom adapter for a system other teams might use, consider contributing it back to the Infrahub Sync repository. + +## Choosing between two valid options + +Sometimes more than one adapter works for the same source system. For example: a Nautobot deployment is reachable through the Nautobot adapter or through the Generic REST API adapter pointed at Nautobot's REST API. Use the dedicated adapter when one exists — it understands the source system's data model and handles edge cases (nested data, authentication) that would otherwise need to be configured manually. + +The Generic REST API adapter is the fallback, not the default. + +## Multiple adapters in one sync project + +A sync project has one source adapter and one destination adapter. To sync data from multiple sources into Infrahub, create one sync project per source. Each project has its own configuration, sync order, and schedule. + +This keeps each sync independent — a failure in one project does not affect others, and each project can be scheduled and operated separately. + +## Related + +- [Generic REST API adapter](./genericrestapi.mdx) +- [Local adapters](./local-adapters.mdx) +- [Create a sync project](../creating-a-sync-project.mdx) +- [Sync instance configuration](../reference/config.mdx) diff --git a/docs/docs/adapters/device42.mdx b/docs/docs/adapters/device42.mdx new file mode 100644 index 0000000..9b67751 --- /dev/null +++ b/docs/docs/adapters/device42.mdx @@ -0,0 +1,24 @@ +--- +title: Device42 adapter +--- + +import ReferenceLink from "../../src/components/Card"; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +## What is Device42? + +[Device42](https://www.device42.com/) is a commercial IT infrastructure and asset management platform. It discovers and inventories physical and virtual infrastructure — servers, network devices, applications, services — and tracks the dependencies between them. + +## Sync directions supported + +- Device42 → Infrahub + +:::info + +The Device42 adapter supports only **one-way synchronization** from Device42 to Infrahub. Writing data back into Device42 is not supported. + +::: + +:::warning Under Construction +::: diff --git a/docs/docs/adapters/nautobot.mdx b/docs/docs/adapters/nautobot.mdx index 84818a6..fbe546a 100644 --- a/docs/docs/adapters/nautobot.mdx +++ b/docs/docs/adapters/nautobot.mdx @@ -167,3 +167,10 @@ As you can see in this example, we can map several paths to the same Infrahub Mo The models available on Nautobot can be find in the `/api/schema/swagger-ui` of your instance. + +## Related guides + +- [Use NetBox or Nautobot with Infrahub](../using-netbox-or-nautobot-with-infrahub.mdx) — running Nautobot and Infrahub side by side +- [Migrate from NetBox or Nautobot](../migrating-from-netbox-or-nautobot.mdx) — gradual migration into Infrahub +- [Schema mapping reference](../reference/schema-mapping.mdx) — mapping syntax with Nautobot examples +- [Choose an adapter](./choosing-an-adapter.mdx) diff --git a/docs/docs/adapters/netbox.mdx b/docs/docs/adapters/netbox.mdx index b517e06..16630b0 100644 --- a/docs/docs/adapters/netbox.mdx +++ b/docs/docs/adapters/netbox.mdx @@ -132,3 +132,10 @@ As you can see in this example, we can map several paths to the same Infrahub Mo The models available on Netbox can be find in the `/api/schema/swagger-ui` of your instance. + +## Related guides + +- [Use NetBox or Nautobot with Infrahub](../using-netbox-or-nautobot-with-infrahub.mdx) — running NetBox and Infrahub side by side +- [Migrate from NetBox or Nautobot](../migrating-from-netbox-or-nautobot.mdx) — gradual migration into Infrahub +- [Schema mapping reference](../reference/schema-mapping.mdx) — mapping syntax with NetBox examples +- [Choose an adapter](./choosing-an-adapter.mdx) diff --git a/docs/docs/adapters/peeringdb.mdx b/docs/docs/adapters/peeringdb.mdx new file mode 100644 index 0000000..40e66df --- /dev/null +++ b/docs/docs/adapters/peeringdb.mdx @@ -0,0 +1,24 @@ +--- +title: PeeringDB adapter +--- + +import ReferenceLink from "../../src/components/Card"; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +## What is PeeringDB? + +[PeeringDB](https://www.peeringdb.com/) is a free, public, user-maintained database of network peering information. It catalogs autonomous systems, internet exchanges, points of presence, and facilities — the data networks use to find and arrange interconnection with one another. + +## Sync directions supported + +- PeeringDB → Infrahub + +:::info + +The PeeringDB adapter supports only **one-way synchronization** from PeeringDB to Infrahub. PeeringDB is a community-maintained reference database; writing data back into it is not supported. + +::: + +:::warning Under Construction +::: diff --git a/docs/docs/contributing.mdx b/docs/docs/contributing.mdx new file mode 100644 index 0000000..533e49a --- /dev/null +++ b/docs/docs/contributing.mdx @@ -0,0 +1,147 @@ +--- +title: Contributing +--- + +This guide covers how to set up a development environment for `infrahub-sync` and contribute to the project. For the release runbook, see [RELEASING.md](https://github.com/opsmill/infrahub-sync/blob/main/RELEASING.md) at the repository root — that's maintainer-only. + +## Prerequisites + +- Python 3.10–3.13 (3.12 recommended) +- [uv](https://docs.astral.sh/uv/) for dependency management +- Git + +## Setting up your development environment + +### Clone the repository + +```bash +git clone https://github.com/opsmill/infrahub-sync.git +cd infrahub-sync +``` + +### Install uv + +If you don't have uv installed, you can install it with: + +```bash +curl -LsSf https://astral.sh/uv/install.sh | sh +``` + +Or see the [uv installation guide](https://docs.astral.sh/uv/getting-started/installation/) for other options. + +### Install dependencies + +```bash +uv sync --group dev +``` + +This installs all runtime and development dependencies defined in `pyproject.toml`. + +### Verify your setup + +```bash +uv run infrahub-sync --help +uv run infrahub-sync list --directory examples/ +``` + +## Development workflow + +Before committing any changes, run the following commands in order: + +```bash +uv run invoke format # Format code with ruff +uv run invoke lint # Lint code with ruff and pylint +uv run mypy infrahub_sync/ --ignore-missing-imports +``` + +### Validate the CLI + +After making changes, verify the CLI still works: + +```bash +uv run infrahub-sync --help +uv run infrahub-sync list --directory examples/ +uv run infrahub-sync generate --name from-netbox --directory examples/ +``` + +### Running tests + +```bash +uv run pytest -q +``` + +## Code standards + +### Python style + +- Python 3.10–3.13 compatible +- Type hints on new or changed code +- Ruff-formatted and lint-clean +- Mypy-checked (do not increase existing error count) +- Public functions and classes require documentation strings +- Raise specific exceptions; avoid broad `except Exception:` + +### Line length + +- Maximum line length: 120 characters (configured in `pyproject.toml`) + +## Documentation + +If you make user-facing changes (CLI flags, configuration options, new adapters), update the documentation. + +### Generate command-line documentation + +```bash +uv run invoke docs.generate +``` + +### Build documentation site + +First-time setup (requires Node.js): + +```bash +cd docs && npm install +``` + +Build the site: + +```bash +uv run invoke docs.docusaurus +``` + +### Lint markdown files + +```bash +npx markdownlint-cli "docs/docs/**/*.{md,mdx}" +npx markdownlint-cli --fix "docs/docs/**/*.{md,mdx}" +``` + +## Adding a new adapter + +1. Create `infrahub_sync/adapters/.py` following existing adapter patterns +2. Add connection configuration schema and an example under `examples/` +3. Provide `list` and `diff` pathways before enabling `sync` +4. Document required environment variables and expected error cases +5. Create a documentation page in `docs/docs/adapters/` +6. Add the adapter to the sidebar in `docs/sidebars.ts` + +## Invoke tasks + +View all available tasks: + +```bash +uv run invoke --list +``` + +Common tasks: + +| Task | Description | +|------|-------------| +| `linter.format-ruff` | Format Python code with ruff | +| `linter.lint-ruff` | Lint Python code with ruff | +| `linter.lint-pylint` | Lint Python code with pylint | +| `linter.lint-yaml` | Lint YAML files with yamllint | +| `docs.generate` | Generate CLI documentation | +| `docs.docusaurus` | Build documentation website | +| `format` | Alias for ruff format | +| `lint` | Run all linters | diff --git a/docs/docs/creating-a-sync-project.mdx b/docs/docs/creating-a-sync-project.mdx new file mode 100644 index 0000000..9e311dd --- /dev/null +++ b/docs/docs/creating-a-sync-project.mdx @@ -0,0 +1,109 @@ +--- +title: Create a sync project +--- + +A sync project is a directory containing a `config.yml` file that defines one synchronization between two systems. Configure four parts: source and destination, sync order, schema mapping, and sync behavior. A Nautobot → Infrahub example runs through each. + +## Define the source and destination + +The `source` and `destination` keys identify which adapter to use on each side of the sync and how to connect to each system. Credentials reference environment variables rather than being embedded in the file. + +```yaml +--- +name: example-sync-task + +source: + name: nautobot + settings: + url: "https://nautobot.example.com" + token: "NAUTOBOT_TOKEN" # This can also be loaded from environment variables + +destination: + name: infrahub + settings: + url: "https://infrahub.example.com" + token: "INFRAHUB_API_TOKEN" # This can also be loaded from environment variables +``` + +For the full list of adapters and their connection parameters, see [Choose an adapter](./adapters/choosing-an-adapter.mdx). + +## Set the sync order + +The `order` key specifies the sequence in which objects should be synchronized. Order matters because some objects depend on others — a device cannot be created until its location, role, and platform already exist in the destination. + +```yaml +order: + - "InfraDevice" + - "InfraInterface" +``` + +## Map the schema fields + +The `schema_mapping` section defines how data is translated from the source's schema into the destination's schema. + +:::info + +- The `name` key in the destination model corresponds to the Infrahub attribute. +- The `mapping` key corresponds to the key in the source payload to use. +- If `reference` is used, it links to a model that has been synchronized prior to this model. + +::: + +```yaml +schema_mapping: + - name: InfraDevice + mapping: "dcim.devices" + identifiers: ["name"] + fields: + - name: "name" + mapping: "name" + - name: "device_type" + mapping: "device_type.display_name" + - name: "manufacturer" + mapping: "device_type.manufacturer.name" + + - name: InfraInterface + mapping: "dcim.interfaces" + identifiers: ["device", "name"] + fields: + - name: "name" + mapping: "name" + - name: "interface_type" + static: "10gbe" + - name: "description" + mapping: "description" + - name: "device" + reference: "InfraDevice" +``` + +In this example, `device_type` and `manufacturer` are attributes of `InfraDevice`. For destination objects that have relationships to other models, the related models must be synchronized first — see how `InfraInterface` references `InfraDevice` via the `reference` key. + +For the full mapping syntax — direct mappings, nested attributes, static values, references, identifiers, filters, transforms, and worked examples — see [Schema mapping reference](./reference/schema-mapping.mdx). + +## Tune sync behavior + +The `diffsync_flags` key controls how the synchronization handles three scenarios: unmatched objects in the destination, unmatched objects in the source, and modified objects. + +```yaml +# Optional: control sync behavior with diffsync flags +diffsync_flags: + - "SKIP_UNMATCHED_DST" # Skip objects in destination that don't exist in source +``` + +:::info Understanding diffsync flags + +Available flags: + +| Flag | Description | +|------|-------------| +| `SKIP_UNMATCHED_DST` | Skip objects in the destination that don't exist in the source (prevents deletion) | +| `SKIP_UNMATCHED_SRC` | Skip objects in the source that don't exist in the destination (prevents creation) | +| `SKIP_MODIFIED` | Skip objects that exist in both systems but have different values (prevents updates) | + +If no flags are specified, `SKIP_UNMATCHED_DST` is used by default — destination objects that don't exist in the source are preserved rather than deleted. + +::: + +For more on customizing sync configuration and troubleshooting, see [Sync instance configuration](./reference/config.mdx). + +When the project is configured, run the sync — see [Run a sync](./running-a-sync.mdx). diff --git a/docs/docs/custom-certificates.mdx b/docs/docs/custom-certificates.mdx new file mode 100644 index 0000000..777e435 --- /dev/null +++ b/docs/docs/custom-certificates.mdx @@ -0,0 +1,32 @@ +--- +title: Use custom CA certificates +--- + +For sync sources or destinations that use TLS certificates signed by an internal CA — common in enterprise networks — install the CA in the system trust store, then point Python's HTTP client at that store. + +## Add the certificate to the system trust store + +Create a subdirectory for the custom CA and copy the root certificate into it: + +```bash +sudo mkdir /usr/local/share/ca-certificates/custom-ca +sudo cp customRCA.crt /usr/local/share/ca-certificates/custom-ca/ +``` + +Then load the certificates into the trusted root store: + +```bash +sudo update-ca-certificates +``` + +The exact paths above are for Debian/Ubuntu systems. On other distributions the trust store lives elsewhere (`/etc/pki/ca-trust/source/anchors/` on RHEL/Fedora, for example) — adapt accordingly. + +## Point Python's HTTP client at the trust store + +Infrahub Sync uses `requests` under the hood, which reads the `REQUESTS_CA_BUNDLE` environment variable to find a custom certificate bundle: + +```bash +export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt +``` + +Set this in the same shell environment where `infrahub-sync` runs — either exported in the user's shell profile, set in the scheduler that runs the sync, or passed in the container environment if running containerized. diff --git a/docs/docs/guides/creation.mdx b/docs/docs/guides/creation.mdx deleted file mode 100644 index e6e4ff5..0000000 --- a/docs/docs/guides/creation.mdx +++ /dev/null @@ -1,112 +0,0 @@ ---- -title: Creating a new sync instance ---- - -This guide will walk you through the steps to create a new sync instance for infrahub-sync, allowing you to synchronize data between your source and destination systems efficiently. - -## Step 1: Define basic configuration - -Start by defining your synchronization source and destination in a YAML configuration file. - -```yaml ---- -name: example-sync-task - -source: - name: nautobot - settings: - url: "https://nautobot.example.com" - token: "NAUTOBOT_TOKEN" # This can also be loaded from environment variables - -destination: - name: infrahub - settings: - url: "https://infrahub.example.com" - token: "INFRAHUB_API_TOKEN" # This can also be loaded from environment variables -``` - -## Step 2: Define synchronization order - -The `order` key specifies the sequence in which objects should be synchronized. This is important because some objects may depend on others. - -```yaml -order: - - "InfraDevice" - - "InfraInterface" -``` - -## Step 3: Configure schema mapping - -The `schema_mapping` section defines how data is mapped from the source to the destination. - -:::info - -- The `name` key in the destination model corresponds to the Infrahub attribute. -- The `mapping` key in the destination model corresponds to the key in the source payload to use. -- If `reference` is used, it links to a model that has been synchronized prior to this model. - -::: - -```yaml -schema_mapping: - - name: InfraDevice - mapping: "dcim.devices" - identifiers: ["name"] - fields: - - name: "name" - mapping: "name" - - name: "device_type" - mapping: "device_type.display_name" - - name: "manufacturer" - mapping: "device_type.manufacturer.name" - - - name: InfraInterface - mapping: "dcim.interfaces" - identifiers: ["device", "name"] - fields: - - name: "name" - mapping: "name" - - name: "interface_type" - static: "10gbe" - - name: "description" - mapping: "description" - - name: "device" - reference: "InfraDevice" -``` - -In this example, `device_type` and `manufacturer` are Attributes of InfraDevice. If you are using another object with a Relationship, you would need to first import those objects and then reference them (like InfraDevice in InfraInterface). - -## Step 4: Configure synchronization behavior - -You can control how the synchronization process handles various scenarios using DiffSync flags. - -```yaml -# Optional: Control sync behavior with diffsync flags -diffsync_flags: - - "SKIP_UNMATCHED_DST" # Skip objects in destination that don't exist in source -``` - -:::info Understanding DiffSync Flags - -DiffSync flags control how the synchronization process handles various scenarios. You can add these flags to your configuration to customize the sync behavior: - -```yaml -diffsync_flags: - - "SKIP_UNMATCHED_DST" # Default if no flags are specified -``` - -Available flags: - -| Flag | Description | -|------|-------------| -| `SKIP_UNMATCHED_DST` | Skip objects in the destination that don't exist in the source (prevents deletion) | -| `SKIP_UNMATCHED_SRC` | Skip objects in the source that don't exist in the destination (prevents creation) | -| `SKIP_MODIFIED` | Skip objects that exist in both systems but have different values (prevents updates) | - -If no flags are specified, `SKIP_UNMATCHED_DST` is used by default, which means objects in the destination that don't exist in the source will be preserved rather than deleted. - -::: - -For more information on customizing your sync configuration and troubleshooting, see the [Sync Instance configuration reference](../reference/config) - -The next steps will be to run the sync, you can check the [Run a sync instance guide](./run) \ No newline at end of file diff --git a/docs/docs/guides/custom-certificates.mdx b/docs/docs/guides/custom-certificates.mdx deleted file mode 100644 index 54a9c95..0000000 --- a/docs/docs/guides/custom-certificates.mdx +++ /dev/null @@ -1,29 +0,0 @@ ---- -title: Support adapters with custom CA certificates ---- - -This guide will walk you through the steps to support adapters with custom CA certificates for infrahub-sync. - -## Step 1: Create a subdirectory for your custom CA certificates - -```bash -sudo mkdir /usr/local/share/ca-certificates/custom-ca -``` - -## Step 2: Copy your root CA custom certificate into the folder you just created - -```bash -cp customRCA.crt /usr/local/share/ca-certificates/custom-ca/ -``` - -## Step 3: Run the following command to load the certificates on your system trusted root certificate store - -```bash -sudo update-ca-certificates -``` - -## Step 4: Use the following command to set `REQUESTS_CA_BUNDLE` environment variable - -```bash -export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt -``` diff --git a/docs/docs/guides/installation.mdx b/docs/docs/installation.mdx similarity index 90% rename from docs/docs/guides/installation.mdx rename to docs/docs/installation.mdx index d8b61c7..1fe1922 100644 --- a/docs/docs/guides/installation.mdx +++ b/docs/docs/installation.mdx @@ -1,5 +1,5 @@ --- -title: Installing Infrahub sync +title: Install Infrahub Sync --- Infrahub Sync is available on [PyPI](https://pypi.org/project/infrahub-sync/) and can be installed using the pip package installer. It is recommended to install the Sync into a virtual environment. diff --git a/docs/docs/migrating-from-netbox-or-nautobot.mdx b/docs/docs/migrating-from-netbox-or-nautobot.mdx new file mode 100644 index 0000000..95f2949 --- /dev/null +++ b/docs/docs/migrating-from-netbox-or-nautobot.mdx @@ -0,0 +1,143 @@ +--- +title: Migrate from NetBox or Nautobot +--- + +Migrate data from NetBox or Nautobot into Infrahub one model at a time. The existing system keeps running throughout, and you decide when — or whether — to retire it. + +Before you start, install Infrahub Sync if you haven't already — see [Install Infrahub Sync](./installation.mdx). + +## The gradual migration approach + +A one-shot cutover from NetBox or Nautobot to Infrahub carries real risk: the new system has to be completely ready, all dependent automation has to switch at the same time, and rollback is hard if something surfaces afterward. The gradual approach removes that risk by syncing data into Infrahub model by model, verifying each step, and migrating dependent workflows on your own schedule. + +The phases: + +1. **Prepare the Infrahub schema.** Define the schema in Infrahub that will hold the migrated data. +2. **Build the sync project.** Create a YAML configuration that maps NetBox or Nautobot models and fields to the corresponding Infrahub models and fields. +3. **Seed Infrahub with initial data.** Run the sync to populate Infrahub with the current state of the source system. +4. **Run in side-by-side mode.** Keep both systems active. Sync runs on a recurring schedule to keep Infrahub current with source-system changes. +5. **Migrate workflows.** Update automation, scripts, and dashboards to read from Infrahub instead of the source system. +6. **Retire the legacy system (optional).** Once all dependent workflows read from Infrahub and the legacy system is no longer the source of truth for any data, decommission it — or keep it running for the workflows where it remains the better tool. + +For long-running side-by-side operation (where the legacy system is not being retired), see [Use NetBox or Nautobot with Infrahub](./using-netbox-or-nautobot-with-infrahub.mdx). The phases below focus on migration specifically. + +## Phase 1 — Prepare the Infrahub schema + +Infrahub uses a custom schema that you define. Before migrating data, decide on the schema Infrahub will use for the data coming over from NetBox or Nautobot. + +The Infrahub schema does not have to mirror the source system one-to-one. Migration is an opportunity to model the data the way you need going forward, not the way the previous tool happened to model it. + +For a concrete starting point, see the example sync projects at `examples/netbox_to_infrahub/` and `examples/nautobot_to_infrahub/` in the [Infrahub Sync repository](https://github.com/opsmill/infrahub-sync). Each references an Infrahub demo schema you can use as a reference when defining your own. Replace its model and field names with the equivalents in your Infrahub schema. + +## Phase 2 — Build the sync project + +A sync project is a directory containing a `config.yml` that defines the source (NetBox or Nautobot), the destination (Infrahub), and the per-model schema mapping. + +### Start from the example + +The Infrahub Sync repository includes working examples at `examples/netbox_to_infrahub/` and `examples/nautobot_to_infrahub/`. Copy the directory matching your source system as the starting point. The examples use the Infrahub demo schema — update model and field names to match the schema you chose in Phase 1. + +### Define sync order + +Models must be synced in dependency order: anything referenced by another model must be synced first. A typical order: + +1. Independent objects: tags, locations, manufacturers, device roles, platforms +2. Rack and site data (depends on locations) +3. Devices (depends on locations, roles, manufacturers, platforms) +4. Interfaces (depends on devices) +5. IP addresses and prefixes (depends on interfaces, VRFs) +6. VLANs and VLAN groups + +The exact order depends on your schema. See [Sync instance configuration](./reference/config.mdx) for the syntax of the `order` key. + +### Map fields between the source system and Infrahub + +For each model, define how source fields map to Infrahub fields. The schema mapping section of the YAML handles direct field mappings, identifier mappings, references to other models, and static values. For detail on each mapping pattern, see [Schema mapping reference](./reference/schema-mapping.mdx). + +## Phase 3 — Seed Infrahub with initial data + +Run `infrahub-sync diff` to verify what the sync will do before applying changes: + +```shell +infrahub-sync diff --name --directory +``` + +The diff prints the proposed changes to the terminal without modifying Infrahub. Review it model by model: + +- Are the right models being created in Infrahub? +- Are field values being mapped correctly? +- Are relationships between models being resolved correctly? +- Are there any unexpected creates, updates, or deletes? + +When the diff looks correct, run the sync: + +```shell +infrahub-sync sync --name --directory +``` + +The sync applies changes in the order defined by the project's `order` key. Independent models are created first, then dependent models, then models that reference earlier ones. + +### Migrate one model at a time + +For the first sync, enable only the most independent models (tags, locations). Verify the data in Infrahub, then add the next layer of models (racks, devices), run again, and so on. This keeps the diff output reviewable and isolates any mapping issues to the model that caused them. + +## Phase 4 — Run in side-by-side mode + +After the initial seed, schedule the sync to run on a recurring cadence so Infrahub stays current with changes in the source system. The cadence depends on how often the source data changes — hourly, daily, or on-demand are all reasonable. + +Infrahub Sync has no built-in scheduler. Use cron, CI, Prefect, or another workflow tool — see [Schedule sync runs](./orchestration.mdx). + +### Default behavior preserves Infrahub-only data + +By default, the `SKIP_UNMATCHED_DST` flag is enabled — Infrahub objects without a corresponding object in the source system are left alone. This is the safe default. Add Infrahub-only data (new schemas, design objects, intent data) without worrying that a sync run will delete it. + +To change this behavior — for example, removing Infrahub objects when they're deleted in the source system — adjust the `diffsync_flags` in the sync configuration. See [Sync instance configuration](./reference/config.mdx). + +### Decide on conflict policy early + +During side-by-side operation, the same model may be edited in both tools. Decide upfront which tool is authoritative for each model and document the decision in the project configuration. If both tools allow edits to the same model, conflicts will eventually occur — and Infrahub Sync's default behavior is for the source system to win. + +## Phase 5 — Migrate workflows + +As Infrahub becomes a reliable mirror of the source system, update dependent automation to read from Infrahub instead. This is the longest phase of a migration and typically happens workflow by workflow: + +- Identify each script, pipeline, or dashboard that reads from the source system. +- Update it to read from Infrahub (using the Python SDK, infrahubctl, or GraphQL). +- Verify the workflow behaves correctly with Infrahub as the source. +- Once you're confident, retire the source-reading version of the workflow. + +For each migrated workflow, you can also stop writing to the source system for the affected data — at which point that data type is fully migrated. + +## Phase 6 — Retire the legacy system (optional) + +Once all dependent workflows read from Infrahub and the legacy system is no longer the source of truth for any data: + +- **Decommission the legacy system entirely** if it's no longer needed. +- **Keep the legacy system running for specific workflows** (rack elevation, cable tracing, or any other capability you prefer there). Reconfigure the sync to keep the legacy system in sync with Infrahub for those workflows, or stop syncing entirely if the data does not need to flow. + +Many teams reach a stable side-by-side state and stay there indefinitely. See [Use NetBox or Nautobot with Infrahub](./using-netbox-or-nautobot-with-infrahub.mdx). + +## Differences between migrating from NetBox versus Nautobot + +The same phases apply for both source systems. The mechanical differences: + +- Use the [NetBox adapter](./adapters/netbox.mdx) for NetBox sources and the [Nautobot adapter](./adapters/nautobot.mdx) for Nautobot sources. +- Start from `examples/netbox_to_infrahub/` for NetBox, `examples/nautobot_to_infrahub/` for Nautobot. +- Field names and nested structures differ between the two systems — for example, Nautobot status fields are objects while NetBox status fields are strings. The schema mapping work is source-specific, but the overall approach is identical. + +## Common issues + +- **Sync runs slowly on large datasets.** Initial syncs of large source-system installations can take a while. Sync performance is an active area of development. Recurring sync runs are typically fast — only deltas are applied. +- **A model is being created in Infrahub that should not be.** Check the sync project's `order` key and per-model filters — the model may be enabled when it shouldn't be, or a reference from another model may be pulling it in. +- **Field values are not mapping correctly.** Check the schema mapping section of the YAML. Common causes: a source field has a different name than expected, or a relationship reference is not resolving because the referenced model was not synced first. +- **Custom fields in the source system are not appearing in Infrahub.** NetBox and Nautobot custom fields require explicit mapping in the YAML — they are not synced automatically. + +## Related + +- [Use NetBox or Nautobot with Infrahub](./using-netbox-or-nautobot-with-infrahub.mdx) +- [Create a sync project](./creating-a-sync-project.mdx) +- [Run a sync](./running-a-sync.mdx) +- [Schema mapping reference](./reference/schema-mapping.mdx) +- [Schedule sync runs](./orchestration.mdx) +- [NetBox adapter](./adapters/netbox.mdx) +- [Nautobot adapter](./adapters/nautobot.mdx) diff --git a/docs/docs/orchestration.mdx b/docs/docs/orchestration.mdx new file mode 100644 index 0000000..6f43373 --- /dev/null +++ b/docs/docs/orchestration.mdx @@ -0,0 +1,96 @@ +--- +title: Schedule sync runs +--- + +Infrahub Sync runs as a single CLI command. Schedule it with whatever tooling already runs scheduled jobs in your environment — cron, CI, Prefect, Dagster, or a homegrown runner. + +## Common orchestration options + +Any tool that can run a CLI command on a schedule and capture its output works. Pick whichever fits your existing operational model. + +### Cron + +The most direct option. A cron entry runs `infrahub-sync sync` on a defined schedule. Output and exit codes are captured by your normal cron logging. + +**When this works well:** + +- You already operate a cron host for scheduled jobs. +- The sync is straightforward — one project, one schedule, no dependencies on other jobs. +- Failure handling can be reactive (logs are reviewed when something looks off). + +**Trade-offs:** No native retry on failure, no visibility beyond logs, no built-in alerting. For mission-critical syncs, wrap the cron entry in a script that handles retries and alerts. + +### CI jobs (GitHub Actions, GitLab CI, etc.) + +Define the sync as a scheduled workflow in your CI system. CI provides run history, log retention, secret management, and notification on failure. + +**When this works well:** + +- You treat infrastructure data movement as part of your software delivery pipeline. +- You want run history and structured failure notifications from tooling you already operate. +- Sync project configurations live in a git repository alongside other infrastructure code. + +**Trade-offs:** CI systems are designed for short-lived jobs. For very large or long-running syncs, watch out for job timeout limits. + +### Prefect + +Prefect is a Python-based workflow orchestrator. Wrap a sync run in a Prefect flow to get retry policies, dependency management, observability through the Prefect UI, and the ability to compose syncs with other Python tasks. + +**When this works well:** + +- You already use Prefect for other infrastructure or data workflows. +- Syncs need to be composed with other tasks — for example, run a sync, then trigger downstream automation. +- You want centralized observability across many sync projects. + +**Trade-offs:** Prefect itself has to be operated. If you don't already run it, this adds operational surface area. + +### Dagster, Airflow, and other workflow engines + +Any workflow engine that can run a CLI command works the same way as Prefect. Pick based on what you already use and what other workloads share the orchestration platform. + +### Event-driven execution + +Some sync use cases are better triggered by an event than by a schedule. For example: "when a device is created in Infrahub, sync its details from the inventory system." For event-driven patterns, Infrahub's trigger event system can call `infrahub-sync` directly instead of running on a fixed cadence. + +Event-driven sync is the exception, not the rule. Most data movement is well-served by a recurring schedule. Use event-driven patterns when the data volume is small, the freshness requirement is high, or the source system is itself event-driven. + +## Choosing a cadence + +Match the cadence to the rate at which source data changes and your tolerance for staleness: + +- **Every few minutes** — for high-change-rate sources that need near-real-time freshness. Verify the sync run completes faster than the cadence to avoid overlap. +- **Hourly** — a common default for active infrastructure data. +- **Daily** — appropriate for slower-moving data or for syncs where the next run can absorb a missed one. +- **On demand** — for migrations, audits, or one-off seeding rather than steady-state operation. + +Each sync run calculates a fresh diff and applies only deltas, so running more often does not multiply work. The constraint is the time the sync takes, not the work it does. + +## Observability and failure handling + +Whatever scheduling tool you use, plan for these operational concerns: + +- **Logging.** Infrahub Sync emits structured logs via `structlog`. Pipe the output to your log aggregator (Splunk, Datadog, Loki, ELK, etc.). +- **Failure detection.** The CLI returns a non-zero exit code on failure. Wire the scheduling tool to catch this and alert the appropriate channel. +- **Idempotency.** Sync runs are idempotent. If a run fails partway through, re-running it calculates a fresh diff against the current destination state and applies what is still outstanding. Retries on failure are safe. +- **Run isolation.** Avoid overlapping runs of the same sync project. Set the cadence longer than the run time, or have the scheduler skip a run if the previous one is still active. +- **Sync project versioning.** Store sync project directories in version control. Tag or release configuration changes the same way you handle infrastructure code. + +## Running Infrahub Sync in a container + +To run Infrahub Sync from any orchestrator while keeping a single Python environment under your control, package it as a container. Build the image with `infrahub-sync` installed and the sync project directory included, set the sync command as the entrypoint, and pass credentials via environment variables. + +Infrahub Sync does not ship a reference container image — examples in the [OpsMill GitHub repository](https://github.com/opsmill/infrahub-sync) show common patterns to adapt. + +## What's not in Infrahub Sync (and what to use instead) + +- **Built-in scheduler.** Use one of the options above. +- **Sync history dashboard.** The scheduling tool typically provides this — CI run history, Prefect UI, or aggregated logs. +- **Built-in retry policies.** Configure retries in the scheduling tool. Runs are idempotent, so retrying is safe. +- **Alerting on failure.** Configure alerts in the scheduling tool or log aggregator. + +## Related + +- [Run a sync](./running-a-sync.mdx) +- [Create a sync project](./creating-a-sync-project.mdx) +- [CLI reference](./reference/cli.mdx) +- [Migrate from NetBox or Nautobot](./migrating-from-netbox-or-nautobot.mdx) diff --git a/docs/docs/readme.mdx b/docs/docs/readme.mdx index 4ffc5b4..0e6bf45 100644 --- a/docs/docs/readme.mdx +++ b/docs/docs/readme.mdx @@ -2,7 +2,7 @@ title: Infrahub Sync --- -Infrahub Sync is a Python package that synchronizes data between Infrahub and external infrastructure systems. It connects to NetBox, Nautobot, IP Fabric, Slurp’it, Cisco ACI, ServiceNow-style systems, and other tools through a library of pre-built adapters, and handles both the communication with each system and the translation of data between different schemas. Sync projects are defined declaratively in YAML; the CLI generates the adapter code, calculates the diff, and applies changes. +Infrahub Sync synchronizes infrastructure data between Infrahub and external systems — NetBox, Nautobot, IP Fabric, Slurp’it, Cisco ACI, Peering Manager, ServiceNow-style CMDBs, and any system with a REST/JSON API. Use it to migrate from a legacy system of record, run two systems side-by-side, or keep Infrahub in sync with another source of truth. Define a sync project declaratively in YAML, and the CLI generates the adapter code, calculates the diff, and applies only the deltas on each run. Infrahub Sync is open source under Apache 2.0, distributed on [PyPI](https://pypi.org/project/infrahub-sync/), and maintained on [GitHub](https://github.com/opsmill/infrahub-sync). @@ -29,7 +29,7 @@ Three CLI commands operate on a sync project: `generate`, `diff`, and `sync`. Each sync project consists of a directory and a `config.yml` describing the sync. The configuration specifies the source adapter and destination adapter with their connection details, the order in which models should be synchronized, and how each source field maps to a destination field. Credentials reference environment variables rather than being embedded in the file. -→ [Creating a new sync instance](./guides/creation.mdx) · [Sync instance configuration](./reference/config.mdx) +→ [Create a sync project](./creating-a-sync-project.mdx) · [Sync instance configuration](./reference/config.mdx) ### Generate the adapter code @@ -45,7 +45,7 @@ Each sync project consists of a directory and a `config.yml` describing the sync Three `diffsync_flags` (`SKIP_UNMATCHED_DST` by default, `SKIP_UNMATCHED_SRC`, `SKIP_MODIFIED`) and per-mapping filters control what each run is allowed to change. -→ [Run a sync instance](./guides/run.mdx) · [Sync CLI](./reference/cli.mdx) +→ [Run a sync](./running-a-sync.mdx) · [Sync CLI](./reference/cli.mdx) ## Who it's for @@ -53,13 +53,13 @@ Three `diffsync_flags` (`SKIP_UNMATCHED_DST` by default, `SKIP_UNMATCHED_SRC`, ` Data lives in NetBox, Nautobot, IP Fabric, or another tool, and the team is adopting Infrahub. Infrahub Sync provides a path that does not require all teams to move at the same time, and that does not require writing integration code. -→ [Installing Infrahub Sync](./guides/installation.mdx) · [Creating a new sync instance](./guides/creation.mdx) +→ [Install Infrahub Sync](./installation.mdx) · [Create a sync project](./creating-a-sync-project.mdx) ### Operating Infrahub at steady state Infrahub is deployed and needs to stay current with the other systems the team uses — IPAM, ITSM, monitoring, network discovery, or in-house databases. One sync project per source, run on the cadence the environment requires. -→ [Creating a new sync instance](./guides/creation.mdx) · [Run a sync instance](./guides/run.mdx) +→ [Create a sync project](./creating-a-sync-project.mdx) · [Run a sync](./running-a-sync.mdx) ### Building inventory from network discovery @@ -98,10 +98,10 @@ Infrahub is populated from what is actually deployed in the network rather than - A running [Infrahub](https://github.com/opsmill/infrahub) instance - Python 3.10–3.13 - Credentials and network access for the source and destination systems -2. **Install Infrahub Sync.** See [Installing Infrahub Sync](./guides/installation.mdx) for the full setup steps. The short version: `pip install infrahub-sync` into a virtual environment. +2. **Install Infrahub Sync.** See [Install Infrahub Sync](./installation.mdx) for the full setup steps. The short version: `pip install infrahub-sync` into a virtual environment. 3. **Choose your starting point.** - - Setting up a sync project for the first time? → [Creating a new sync instance](./guides/creation.mdx) - - Running an existing project? → [Run a sync instance](./guides/run.mdx) + - Setting up a sync project for the first time? → [Create a sync project](./creating-a-sync-project.mdx) + - Running an existing project? → [Run a sync](./running-a-sync.mdx) ## Common questions @@ -127,12 +127,12 @@ Yes. Each sync project is independent — a separate directory, configuration, a | What you want to do | Where to go | |---|---| -| Set up your environment | [Installing Infrahub Sync](./guides/installation.mdx) | -| Configure a sync project | [Creating a new sync instance](./guides/creation.mdx) · [Sync instance configuration](./reference/config.mdx) | -| Run a sync | [Run a sync instance](./guides/run.mdx) | +| Set up your environment | [Install Infrahub Sync](./installation.mdx) | +| Configure a sync project | [Create a sync project](./creating-a-sync-project.mdx) · [Sync instance configuration](./reference/config.mdx) | +| Run a sync | [Run a sync](./running-a-sync.mdx) | | CLI reference | [Sync CLI](./reference/cli.mdx) | | All adapters | See the **Adapters** section in the sidebar | -| Custom CA certificates | [Support adapters with custom CA certificates](./guides/custom-certificates.mdx) | +| Custom CA certificates | [Use custom CA certificates](./custom-certificates.mdx) | | Build a custom adapter | [Local Adapters](./adapters/local-adapters.mdx) | -| Contribute | [Development guide](./development.mdx) | +| Contribute | [Contributing guide](./contributing.mdx) | | Source code | [github.com/opsmill/infrahub-sync](https://github.com/opsmill/infrahub-sync) | diff --git a/docs/docs/reference/config.mdx b/docs/docs/reference/config.mdx index cbf1044..afa1647 100644 --- a/docs/docs/reference/config.mdx +++ b/docs/docs/reference/config.mdx @@ -1,5 +1,5 @@ --- -title: Sync configuration file +title: Sync instance configuration --- import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; @@ -44,7 +44,7 @@ Configuration for source and destination adapters. ### Schema Mapping -Defines the mappings from source to destination schemas. +Defines the mappings from source to destination schemas. The tables below are the structural field reference. For the mapping syntax in depth — direct mappings, references, identifiers, filters, transforms, and worked NetBox and Nautobot examples — see [Schema mapping reference](./schema-mapping.mdx). #### Mapping models @@ -74,24 +74,7 @@ Defines the mappings from source to destination schemas. | operation | string | Which filters will be applied on the field. Available operations are described below. | No | | value | string | The expected value linked to the operation. | Yes except for both is_empty and is_not_empty operation | -###### Available filter operations - -| Operation | Description | -| --------- | ----------- | -| `==` | Checks if the field value is equal to the provided value. | -| `!=` | Checks if the field value is not equal to the provided value. | -| `>` | Checks if the field value is greater than the provided value (after converting both to integers if necessary). | -| `<` | Checks if the field value is less than the provided value (after converting both to integers if necessary). | -| `>=` | Checks if the field value is greater than or equal to the provided value (after converting both to integers if necessary). | -| `<=` | Checks if the field value is less than or equal to the provided value (after converting both to integers if necessary). | -| `in` | Checks if the field value is within the provided list or string. | -| `not in` | Checks if the field value is not within the provided list or string. | -| `contains` | Checks if the provided value is contained within the field value. | -| `not contains` | Checks if the provided value is not contained within the field value. | -| `is_empty` | Checks if the field value is `None` or empty. Does not require a `value` argument. | -| `is_not_empty` | Checks if the field value is not `None` and not empty. Does not require a `value` argument. | -| `regex` | Checks if the field value matches the provided regular expression pattern. | -| `is_ip_within` | Checks if the field value (an IP address) is within the provided IP range using `netutils`. | +For the full list of filter operations (`==`, `!=`, `>`, `<`, `>=`, `<=`, `in`, `not in`, `contains`, `not contains`, `is_empty`, `is_not_empty`, `regex`, `is_ip_within`) with examples, see [Available filter operations](./schema-mapping.mdx#available-filter-operations). ##### Mapping transforms diff --git a/docs/docs/reference/schema-mapping.mdx b/docs/docs/reference/schema-mapping.mdx new file mode 100644 index 0000000..97680b4 --- /dev/null +++ b/docs/docs/reference/schema-mapping.mdx @@ -0,0 +1,310 @@ +--- +title: Schema mapping reference +--- + +The schema mapping section of a sync project's `config.yml` defines how data translates between the source system and the destination system. Worked examples for NetBox → Infrahub and Nautobot → Infrahub appear at the end. + +For the full configuration schema (not just the mapping section), see [Sync instance configuration](./config.mdx). + +## Anatomy of a schema mapping + +Each entry in the `schema_mapping` section maps one model from the source system to one model in the destination system. A mapping entry contains: + +- **Name** — the destination model name. +- **Mapping** — the source model name in the source system. +- **Identifiers** — the field(s) that uniquely identify an object in this model. +- **Fields** — the per-field mappings between source and destination. + +A minimal example: + +```yaml +schema_mapping: + - name: Device + mapping: dcim.device + identifiers: ["name"] + fields: + - name: name + mapping: name + - name: serial + mapping: serial + - name: status + mapping: status.value +``` + +This says: for each `dcim.device` in the source system, create or update a `Device` object in the destination using the `name` field as the unique identifier, and map the `name`, `serial`, and `status.value` fields directly. + +## Field mappings + +Each entry in the `fields` list defines one field mapping. The most common patterns are direct mapping, nested attribute mapping, static values, and references to other models. + +### Direct field mapping + +The source field name maps directly to the destination field name: + +```yaml +- name: name + mapping: name +``` + +This is the most common case. The destination field `name` gets the value of the source field `name`. + +### Renaming fields + +When the destination field has a different name than the source field, list them separately: + +```yaml +- name: device_role + mapping: role.slug +``` + +The destination field `device_role` gets the value of the source's `role.slug`. + +### Nested attribute mapping + +For source data with nested structure, use dotted notation to reach into the nested attribute: + +```yaml +- name: status + mapping: status.value +- name: site + mapping: site.slug +- name: primary_ip + mapping: primary_ip4.address +``` + +### Static values + +When a destination field should always be set to the same value regardless of source data, use `static`: + +```yaml +- name: source_system + static: netbox +``` + +This is useful for fields that record provenance, type discriminators, or any field where the value is determined by the sync configuration rather than the source data. + +### References to other models + +When a destination field is a relationship to another model that is also being synced, use `reference` to point to that model: + +```yaml +- name: device + mapping: device.name + reference: Device +``` + +This says: the value of the source's `device.name` field should resolve to the `Device` object with that name in the destination. The referenced model must be synced before this one — see the `order` key in the project configuration. + +## Identifiers + +The `identifiers` list defines which field(s) uniquely identify an object of this model. Identifiers are used to determine whether an object already exists in the destination and should be updated, or whether a new object should be created. + +### Single-field identifiers + +For most models, a single field — typically `name` — is enough: + +```yaml +identifiers: ["name"] +``` + +### Composite identifiers + +Some models need multiple fields to be uniquely identified. For example, an IP address is unique by address plus VRF: + +```yaml +identifiers: ["address", "vrf"] +``` + +All listed fields must be present and unique together. Each identifier field must also be mapped in the `fields` list. + +## Filters + +Filters control which objects from the source get synced. Apply them at the model level to include or exclude specific objects. + +```yaml +- name: Device + mapping: dcim.device + identifiers: ["name"] + filters: + - field: status.value + operation: "==" + value: active + fields: + - name: name + mapping: name +``` + +This syncs only devices with `status.value == "active"`. Other devices are skipped. + +### Available filter operations + +Infrahub Sync supports the following 14 filter operations: + +| Operation | Description | +| --- | --- | +| `==` | Equal to the value | +| `!=` | Not equal to the value | +| `>` | Greater than (numeric) | +| `<` | Less than (numeric) | +| `>=` | Greater than or equal (numeric) | +| `<=` | Less than or equal (numeric) | +| `in` | Value is in a list or string | +| `not in` | Value is not in a list or string | +| `contains` | Field value contains the given value | +| `not contains` | Field value does not contain the given value | +| `is_empty` | Field is `None` or empty (no `value` argument needed) | +| `is_not_empty` | Field is not `None` and not empty (no `value` argument needed) | +| `regex` | Field matches the regular expression in `value` | +| `is_ip_within` | Field (an IP address) is within the IP range in `value` | + +Multiple filters on the same model are combined with AND — an object must match all filters to be included. + +## Transforms and custom Jinja filters + +For cases where a field value needs to be transformed during the mapping — uppercase a string, parse a date, compute a derived value — apply a `transforms` entry to the mapping. The transform takes a `field` (the target field name) and an `expression` (a Jinja-compatible expression evaluated against the source object). + +```yaml +- name: Device + mapping: dcim.device + identifiers: ["name"] + fields: + - name: name + mapping: name + transforms: + - field: name + expression: "{{ name | upper }}" +``` + +Adapters can also register custom Jinja filters via a `_add_custom_filters` class method on the adapter model. The ACI adapter, for example, includes an `aci_device_name` filter for resolving ACI node IDs to device names. See [Local Adapters](../adapters/local-adapters.mdx) for the implementation pattern, and [Sync instance configuration](./config.mdx) for the full `transforms` syntax. + +## Validating a mapping + +Validate the mapping with the diff command before applying any changes: + +```shell +infrahub-sync diff --name --directory +``` + +The diff prints every proposed create, update, and delete. Review it model by model: + +- Are the field values being mapped correctly? +- Are references resolving to the right objects in the destination? +- Are filters excluding the right objects? +- Are there unexpected creates or deletes? + +Adjust the mapping and re-run the diff until it matches expectations. Only then run the sync. + +--- + +Two common source systems — NetBox and Nautobot — illustrate how the syntax above plays out: + +## Common patterns for NetBox → Infrahub + +The patterns below cover the most common NetBox → Infrahub mappings. Adapt the destination field names to match your Infrahub schema. + +### Sites and locations + +```yaml +- name: Location + mapping: dcim.site + identifiers: ["name"] + fields: + - name: name + mapping: name + - name: slug + mapping: slug + - name: description + mapping: description + - name: status + mapping: status.value +``` + +### Devices with role and platform references + +```yaml +- name: Device + mapping: dcim.device + identifiers: ["name"] + fields: + - name: name + mapping: name + - name: serial + mapping: serial + - name: location + mapping: site.slug + reference: Location + - name: role + mapping: role.slug + reference: DeviceRole + - name: platform + mapping: platform.slug + reference: Platform +``` + +### Interfaces with device reference + +```yaml +- name: Interface + mapping: dcim.interface + identifiers: ["name", "device"] + fields: + - name: name + mapping: name + - name: device + mapping: device.name + reference: Device + - name: enabled + mapping: enabled + - name: type + mapping: type.value +``` + +### IP addresses with VRF reference + +```yaml +- name: IPAddress + mapping: ipam.ip-address + identifiers: ["address", "vrf"] + fields: + - name: address + mapping: address + - name: vrf + mapping: vrf.name + reference: VRF + - name: status + mapping: status.value + - name: description + mapping: description +``` + +### Filtering by tenant or tag + +```yaml +- name: Device + mapping: dcim.device + identifiers: ["name"] + filters: + - field: tenant.slug + operation: "==" + value: production + fields: + - name: name + mapping: name +``` + +## Common patterns for Nautobot → Infrahub + +Nautobot's API and model structure are similar to NetBox but with some differences in field names and nested structure. The mapping patterns above largely apply; the main differences: + +- Nautobot uses `display` instead of `name` in some places. +- Nautobot status fields are objects rather than strings — use `status.name` instead of `status.value`. +- Custom fields and computed fields appear in different sections of the API response. + +Start from the `examples/nautobot_to_infrahub/` directory in the Infrahub Sync repository and adapt from there. + +## Related + +- [Sync instance configuration](./config.mdx) +- [Create a sync project](../creating-a-sync-project.mdx) +- [NetBox adapter](../adapters/netbox.mdx) +- [Nautobot adapter](../adapters/nautobot.mdx) diff --git a/docs/docs/guides/run.mdx b/docs/docs/running-a-sync.mdx similarity index 89% rename from docs/docs/guides/run.mdx rename to docs/docs/running-a-sync.mdx index bddf659..867cb48 100644 --- a/docs/docs/guides/run.mdx +++ b/docs/docs/running-a-sync.mdx @@ -1,15 +1,15 @@ --- -title: Running sync tasks +title: Run a sync --- Learn how to use Infrahub Sync's commands to generate sync adapters, calculate differences, and synchronize data between your source and destination systems. -![Infrahub-Sync process](../media/infrahub_sync_process.excalidraw.svg) +![Infrahub-Sync process](./media/infrahub_sync_process.excalidraw.svg) -::: info +:::info -Before generating the necessary Python code for your sync adapters and models and synchronizing, you need to created a configuration. -To create a new configuration, please refer to the guide [Creating a new Sync Instance](./creation) +Before generating the necessary Python code for your sync adapters and models and synchronizing, you need to create a configuration. +To create a new configuration, please refer to [Create a sync project](./creating-a-sync-project.mdx). ::: diff --git a/docs/docs/using-netbox-or-nautobot-with-infrahub.mdx b/docs/docs/using-netbox-or-nautobot-with-infrahub.mdx new file mode 100644 index 0000000..2955027 --- /dev/null +++ b/docs/docs/using-netbox-or-nautobot-with-infrahub.mdx @@ -0,0 +1,88 @@ +--- +title: Use NetBox or Nautobot with Infrahub +--- + +If you're adopting Infrahub but already have an established system of record in NetBox or Nautobot — with accumulated data, custom fields, and automation built around it — run Infrahub side-by-side with the existing tool and use Infrahub Sync to keep both systems holding the same data. + +This is the most common adoption pattern, whether you plan to eventually consolidate on Infrahub or keep both systems running long-term. + +## The side-by-side pattern + +Both tools continue to operate. The existing system (NetBox or Nautobot) remains the system of record for some data; Infrahub becomes the system of record for the rest. A sync project moves data between them on a defined cadence, so each system holds the data it owns plus a current view of what the other system owns. + +The side-by-side pattern fits when: + +- One team is ready to move to Infrahub, but other teams still depend on the existing system. +- Existing automation pipelines, dashboards, or scripts read from NetBox or Nautobot and need time to migrate. +- The existing system has features with no direct Infrahub equivalent (rack elevation, native cable tracing), and you want to keep using it for those workflows. +- You want to evaluate Infrahub on production data. + +This pattern is also the typical starting point for a gradual migration. See [Migrate from NetBox or Nautobot](./migrating-from-netbox-or-nautobot.mdx) for the migration-specific walkthrough. + +## Choosing which tool owns which data + +Decide which system is authoritative for each type of data before configuring a sync project. Sync behavior on each model is controlled by which side is the source and which is the destination, plus the `diffsync_flags` that govern what each run is allowed to change. + +Common patterns: + +- **The existing system is authoritative for everything during initial evaluation.** Sync runs NetBox → Infrahub or Nautobot → Infrahub. Infrahub mirrors the existing system; edits in Infrahub are not propagated back. This is the safest starting configuration. +- **Infrahub is authoritative for newly modeled data; the existing system stays authoritative for existing data.** The sync project covers only the models that overlap. New data types (intent, design objects, custom schemas) live in Infrahub and are not synced. +- **Each system owns specific models.** For example, the existing system owns rack and cable data; Infrahub owns device intent, IP allocations, and configuration data. Sync moves only the models each system needs to consume. + +Document the ownership decision in the sync project's configuration comments — anyone reading the YAML later needs to understand the boundary. + +## Mapping data between the existing system and Infrahub + +NetBox and Nautobot each have a fixed schema. Infrahub has a custom schema that you define. There is no universal mapping — the YAML configuration depends on the specific Infrahub schema in use. + +For a typical mapping: + +- Start from the example configuration in the [Infrahub Sync repository](https://github.com/opsmill/infrahub-sync) under `examples/netbox_to_infrahub/` or `examples/nautobot_to_infrahub/`. The example references the Infrahub demo schema; replace its model and field names with the equivalents in your Infrahub schema. +- For each source model you want to sync (sites, racks, devices, interfaces, IP addresses, VLANs, etc.), find the equivalent Infrahub model and map fields directly. +- For field name differences (NetBox's `device_role` versus an Infrahub `role` attribute, or Nautobot's `status.name` versus NetBox's `status.value`), the schema mapping section of the YAML handles the translation. +- For relationship differences, use `references` in the YAML to resolve them. See [Sync instance configuration](./reference/config.mdx) for the full syntax. + +For a step-by-step guide on building the schema mapping itself, see [Schema mapping reference](./reference/schema-mapping.mdx). + +## Schema differences to be aware of + +Some NetBox and Nautobot features have no direct Infrahub equivalent today. For each: keep using the existing system for that workflow, or model the capability in Infrahub with available workarounds. + +### Rack elevation views + +NetBox and Nautobot both include a built-in rack elevation visualization. Infrahub does not have a native rack elevation view, and rack capacity calculation (free U positions, weight, power draw) is not automatic. + +If rack elevation is important to your workflow, the most direct path is to keep managing racks in the existing system and exclude rack data from the sync. Alternatively, model rack data in Infrahub with a custom schema and handle capacity calculation through a generator or GraphQL queries. + +### Cable tracing + +NetBox and Nautobot both include a cable tracing feature that follows connections through patch panels and intermediate devices. Infrahub does not have a single-click cable trace view, but the same data is queryable via Infrahub's graph traversal capabilities. For visualization, build a custom artifact or query with GraphQL from external tools. + +### Other differences + +For any feature where the existing system has a direct capability and Infrahub does not, document the decision in the sync project configuration and update internal runbooks. Keeping a NetBox or Nautobot workflow alongside Infrahub is a valid long-term choice. + +## Operational considerations + +- **Run direction matters for conflict handling.** When syncing NetBox → Infrahub or Nautobot → Infrahub, edits made in Infrahub to synced models will be overwritten on the next sync run unless flags are configured otherwise. Make sure everyone on the team understands which side is the source of truth for each model. +- **Use `diff` regularly.** Before applying any sync — and especially during initial setup — run `infrahub-sync diff` to verify what would change. The diff is read-only. +- **Schedule syncs on a cadence that matches data change rates.** If the source data changes hourly, sync hourly. If it changes daily, sync daily. Infrahub Sync has no built-in scheduler — use cron, CI, or a workflow tool. See [Schedule sync runs](./orchestration.mdx). +- **Set explicit ownership boundaries.** If both tools allow edits to the same model, conflicts will eventually occur. Decide upfront which tool is authoritative; avoid bi-directional syncing of the same model unless conflict resolution is well-defined. + +## When to move toward full migration + +The side-by-side pattern is fine as a long-term steady state, but most teams eventually consolidate on Infrahub once it has proven itself and the gaps that kept the existing system in place have been addressed. Signals that you're ready to migrate fully: + +- All workflows that depended on NetBox-only or Nautobot-only features have been replaced or moved into Infrahub. +- Existing automation pipelines have been updated to read from Infrahub. +- Data ownership in Infrahub has been stable for a meaningful period — no surprises in what's there. + +At that point, see [Migrate from NetBox or Nautobot](./migrating-from-netbox-or-nautobot.mdx) for the full migration walkthrough. + +## Related + +- [Migrate from NetBox or Nautobot](./migrating-from-netbox-or-nautobot.mdx) +- [Create a sync project](./creating-a-sync-project.mdx) +- [Schema mapping reference](./reference/schema-mapping.mdx) +- [NetBox adapter](./adapters/netbox.mdx) +- [Nautobot adapter](./adapters/nautobot.mdx) diff --git a/docs/docusaurus.config.ts b/docs/docusaurus.config.ts index e6110fd..82846e6 100644 --- a/docs/docusaurus.config.ts +++ b/docs/docusaurus.config.ts @@ -27,6 +27,36 @@ const config: Config = { locales: ['en'], }, + plugins: [ + [ + '@docusaurus/plugin-client-redirects', + { + redirects: [ + { + from: '/guides/installation', + to: '/installation', + }, + { + from: '/guides/creation', + to: '/creating-a-sync-project', + }, + { + from: '/guides/run', + to: '/running-a-sync', + }, + { + from: '/guides/custom-certificates', + to: '/custom-certificates', + }, + { + from: '/development', + to: '/contributing', + }, + ], + }, + ], + ], + presets: [ [ "classic", diff --git a/docs/package-lock.json b/docs/package-lock.json index a2f439f..c18078d 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -9,6 +9,7 @@ "version": "0.0.0", "dependencies": { "@docusaurus/core": "^3.10.0", + "@docusaurus/plugin-client-redirects": "3.10.0", "@docusaurus/preset-classic": "^3.10.0", "@mdx-js/react": "^3.0.0", "clsx": "^2.0.0", @@ -3608,6 +3609,30 @@ "react-dom": "*" } }, + "node_modules/@docusaurus/plugin-client-redirects": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/@docusaurus/plugin-client-redirects/-/plugin-client-redirects-3.10.0.tgz", + "integrity": "sha512-P+VLoLoZTc74so8+IbsaPZ33/mkf2BWL1CYXQpPRkl0v1QVCN2CgfsZY/8QtbYjQnx2upXUnv45abDhNcSggNw==", + "license": "MIT", + "dependencies": { + "@docusaurus/core": "3.10.0", + "@docusaurus/logger": "3.10.0", + "@docusaurus/utils": "3.10.0", + "@docusaurus/utils-common": "3.10.0", + "@docusaurus/utils-validation": "3.10.0", + "eta": "^2.2.0", + "fs-extra": "^11.1.1", + "lodash": "^4.17.21", + "tslib": "^2.6.0" + }, + "engines": { + "node": ">=20.0" + }, + "peerDependencies": { + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + } + }, "node_modules/@docusaurus/plugin-content-blog": { "version": "3.10.0", "resolved": "https://registry.npmjs.org/@docusaurus/plugin-content-blog/-/plugin-content-blog-3.10.0.tgz", diff --git a/docs/package.json b/docs/package.json index 21a0d76..7cc4328 100644 --- a/docs/package.json +++ b/docs/package.json @@ -16,6 +16,7 @@ }, "dependencies": { "@docusaurus/core": "^3.10.0", + "@docusaurus/plugin-client-redirects": "3.10.0", "@docusaurus/preset-classic": "^3.10.0", "@mdx-js/react": "^3.0.0", "clsx": "^2.0.0", diff --git a/docs/sidebars.ts b/docs/sidebars.ts index bf159cc..52fd330 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -3,20 +3,32 @@ import type {SidebarsConfig} from '@docusaurus/plugin-content-docs'; const sidebars: SidebarsConfig = { syncSidebar: [ 'readme', + { + type: 'category', + label: 'Get started', + items: [ + 'installation', + 'creating-a-sync-project', + 'running-a-sync', + ], + }, { type: 'category', label: 'Guides', items: [ - 'guides/installation', - 'guides/creation', - 'guides/run', + 'using-netbox-or-nautobot-with-infrahub', + 'migrating-from-netbox-or-nautobot', + 'orchestration', + 'custom-certificates', ], }, { type: 'category', label: 'Adapters', items: [ + 'adapters/choosing-an-adapter', 'adapters/aci', + 'adapters/device42', 'adapters/genericrestapi', 'adapters/infrahub', 'adapters/ipfabric', @@ -26,6 +38,7 @@ const sidebars: SidebarsConfig = { 'adapters/netbox', 'adapters/observium', 'adapters/peering-manager', + 'adapters/peeringdb', 'adapters/prometheus', 'adapters/slurpit', ], @@ -35,10 +48,11 @@ const sidebars: SidebarsConfig = { label: 'Reference', items: [ 'reference/config', + 'reference/schema-mapping', 'reference/cli', ], }, - 'development', + 'contributing', ] };