diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 7cbe767..872f6ad 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,100 +1,53 @@ -name: Publish to crates.io +name: Publish on: release: types: [published] workflow_dispatch: - inputs: - dry_run: - description: 'Dry run (no actual publish)' - required: false - default: 'true' - type: boolean - crate: - description: 'Crate to publish (all, fetchkit, fetchkit-cli)' - required: false - default: 'all' - type: choice - options: - - all - - fetchkit - - fetchkit-cli permissions: contents: read env: CARGO_TERM_COLOR: always - RUST_BACKTRACE: 1 jobs: - test: - name: Test before publish - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - name: Run tests - run: cargo test --workspace - - name: Check formatting - run: cargo fmt --all -- --check - - name: Clippy - run: cargo clippy --workspace --all-targets -- -D warnings - publish-fetchkit: name: Publish fetchkit - needs: test runs-on: ubuntu-latest - if: >- - github.event_name == 'release' || - (github.event_name == 'workflow_dispatch' && - (github.event.inputs.crate == 'all' || github.event.inputs.crate == 'fetchkit')) steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - name: Verify crate can be packaged - run: cargo package -p fetchkit --allow-dirty - - - name: Publish fetchkit (dry run) - if: github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true' - run: cargo publish -p fetchkit --dry-run - - - name: Publish fetchkit - if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'false') + - name: Verify version matches tag + run: | + TAG_VERSION="${{ github.event.release.tag_name }}" + if [ -n "$TAG_VERSION" ]; then + TAG_VERSION="${TAG_VERSION#v}" + CARGO_VERSION=$(grep '^version' Cargo.toml | head -1 | sed 's/.*"\(.*\)".*/\1/') + if [ "$TAG_VERSION" != "$CARGO_VERSION" ]; then + echo "Error: Tag version does not match Cargo.toml version" + exit 1 + fi + fi + + - name: Publish fetchkit to crates.io run: cargo publish -p fetchkit env: CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} publish-fetchkit-cli: name: Publish fetchkit-cli - needs: [test, publish-fetchkit] runs-on: ubuntu-latest - if: >- - github.event_name == 'release' || - (github.event_name == 'workflow_dispatch' && - (github.event.inputs.crate == 'all' || github.event.inputs.crate == 'fetchkit-cli')) + needs: publish-fetchkit steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - # Wait for crates.io index to update after fetchkit publish - - name: Wait for crates.io index - if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'false') + - name: Wait for crates.io index update run: sleep 30 - - name: Verify crate can be packaged - run: cargo package -p fetchkit-cli --allow-dirty - - - name: Publish fetchkit-cli (dry run) - if: github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true' - run: cargo publish -p fetchkit-cli --dry-run - - - name: Publish fetchkit-cli - if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'false') + - name: Publish fetchkit-cli to crates.io run: cargo publish -p fetchkit-cli env: CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 711ddb1..332d226 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,87 +4,63 @@ on: push: branches: - main + workflow_dispatch: + +permissions: + contents: write + actions: write jobs: release: + name: Create Release runs-on: ubuntu-latest - # Only run if commit message matches release pattern - if: ${{ startsWith(github.event.head_commit.message, 'chore(release): prepare v') }} - - permissions: - contents: write + if: "${{ github.event_name == 'workflow_dispatch' || startsWith(github.event.head_commit.message, 'chore(release): prepare v') }}" steps: - name: Checkout uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Extract version from commit message + - name: Extract version id: version run: | - # Extract version from commit message like "chore(release): prepare v0.1.0" - VERSION=$(echo "${{ github.event.head_commit.message }}" | grep -oP 'prepare v\K[0-9]+\.[0-9]+\.[0-9]+') + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + VERSION=$(grep '^version' Cargo.toml | head -1 | sed 's/.*"\(.*\)".*/\1/') + else + COMMIT_MSG="${{ github.event.head_commit.message }}" + VERSION=$(echo "$COMMIT_MSG" | sed -n 's/.*prepare v\([0-9]*\.[0-9]*\.[0-9]*\).*/\1/p') + fi if [ -z "$VERSION" ]; then - echo "::error::Could not extract version from commit message" + echo "Error: Could not determine version" exit 1 fi echo "version=$VERSION" >> $GITHUB_OUTPUT echo "tag=v$VERSION" >> $GITHUB_OUTPUT - echo "Extracted version: $VERSION" - - - name: Check if tag already exists - id: check_tag - run: | - if git rev-parse "v${{ steps.version.outputs.version }}" >/dev/null 2>&1; then - echo "::error::Tag v${{ steps.version.outputs.version }} already exists" - exit 1 - fi - echo "Tag does not exist, proceeding..." - - name: Verify Cargo.toml version matches + - name: Verify version matches Cargo.toml run: | - CARGO_VERSION=$(grep '^version = ' Cargo.toml | head -1 | sed 's/version = "\(.*\)"/\1/') - if [ "$CARGO_VERSION" != "${{ steps.version.outputs.version }}" ]; then - echo "::error::Cargo.toml version ($CARGO_VERSION) does not match release version (${{ steps.version.outputs.version }})" + CARGO_VERSION=$(grep '^version' Cargo.toml | head -1 | sed 's/.*"\(.*\)".*/\1/') + if [ "${{ steps.version.outputs.version }}" != "$CARGO_VERSION" ]; then + echo "Error: Commit version does not match Cargo.toml version" exit 1 fi - echo "Cargo.toml version matches: $CARGO_VERSION" - - name: Extract release notes from CHANGELOG.md + - name: Extract release notes from CHANGELOG id: changelog run: | VERSION="${{ steps.version.outputs.version }}" - - # Extract the section for this version from CHANGELOG.md - # Matches from "## [X.Y.Z]" until the next "## [" or end of significant content - NOTES=$(awk -v ver="$VERSION" ' - /^## \[/ { - if (found) exit - if (index($0, "[" ver "]")) found=1 - next - } - found && /^## \[/ { exit } - found { print } - ' CHANGELOG.md) - - if [ -z "$NOTES" ]; then - echo "::warning::No changelog entry found for version $VERSION" - NOTES="Release v$VERSION" - fi - - # Write to file to preserve formatting + NOTES=$(awk "/^## \[$VERSION\]/{found=1; next} /^## \[/{if(found) exit} found{print}" CHANGELOG.md) echo "$NOTES" > release_notes.md - echo "Release notes extracted:" - cat release_notes.md - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ steps.version.outputs.tag }} + name: Release ${{ steps.version.outputs.tag }} + body_path: release_notes.md + draft: false + prerelease: false + + - name: Trigger publish workflow + run: gh workflow run publish.yml env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh release create "v${{ steps.version.outputs.version }}" \ - --title "v${{ steps.version.outputs.version }}" \ - --notes-file release_notes.md \ - --target "${{ github.sha }}" - - echo "Created release v${{ steps.version.outputs.version }}" diff --git a/AGENTS.md b/AGENTS.md index 773c434..5349b5f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -98,19 +98,20 @@ See `docs/release-process.md` for full release process documentation. Quick summary: 1. Human asks agent: "Create release v0.2.0" -2. Agent updates CHANGELOG.md, Cargo.toml version, creates PR +2. Agent updates CHANGELOG.md (with Highlights + What's Changed), Cargo.toml version, creates PR 3. Human reviews and merges PR to main -4. CI creates GitHub Release (release.yml) -5. CI publishes to crates.io (publish.yml) +4. CI creates GitHub Release via `softprops/action-gh-release` (release.yml) +5. release.yml triggers publish.yml +6. CI publishes `fetchkit` then `fetchkit-cli` to crates.io (publish.yml) Workflows: -- `.github/workflows/release.yml` - Creates GitHub Release on merge -- `.github/workflows/publish.yml` - Publishes to crates.io on GitHub Release +- `.github/workflows/release.yml` - Creates GitHub Release on merge or manual dispatch +- `.github/workflows/publish.yml` - Publishes to crates.io on GitHub Release or manual dispatch Requirements: - `CARGO_REGISTRY_TOKEN` secret must be configured in repo settings -Note: `fetchkit-python` is not published to crates.io (uses PyPI distribution instead). +Note: `fetchkit-python` is not published to crates.io (`publish = false`). Uses PyPI distribution instead. ### Cloud Agent environments diff --git a/CHANGELOG.md b/CHANGELOG.md index 3989698..f283ae3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,16 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.1.0] - 2026-02-12 + +### Highlights + +- AI-friendly web content fetching with HTML-to-Markdown and HTML-to-Text conversion +- CLI and MCP server for AI tool integration +- Pluggable fetcher system for URL-specific handling +- Python bindings via PyO3 + ### What's Changed -- feat: add pluggable fetcher system for URL-specific handling ([#9](https://github.com/everruns/fetchkit/pull/9)) by @chaliy -- docs: add LangChain example for MCP integration ([#8](https://github.com/everruns/fetchkit/pull/8)) by @chaliy -- refactor(cli): unified md-first output format ([#7](https://github.com/everruns/fetchkit/pull/7)) by @chaliy -- docs: clarify test classification in AGENTS.md ([#6](https://github.com/everruns/fetchkit/pull/6)) by @chaliy -- docs: add cloud agent env and complete AGENTS.md placeholders ([#5](https://github.com/everruns/fetchkit/pull/5)) by @chaliy -- refactor: rename project from webfetch to fetchkit ([#4](https://github.com/everruns/fetchkit/pull/4)) by @chaliy -- docs: add comprehensive README with installation and usage guide ([#3](https://github.com/everruns/fetchkit/pull/3)) by @chaliy -- feat: implement webfetch library, CLI, MCP server, and Python bindings ([#1](https://github.com/everruns/fetchkit/pull/1)) by @chaliy -- feat: add initial webfetch spec and guidance by @chaliy - -[Unreleased]: https://github.com/everruns/fetchkit/compare/HEAD...HEAD +* feat: add pluggable fetcher system for URL-specific handling ([#9](https://github.com/everruns/fetchkit/pull/9)) by @chaliy +* docs: add LangChain example for MCP integration ([#8](https://github.com/everruns/fetchkit/pull/8)) by @chaliy +* refactor(cli): unified md-first output format ([#7](https://github.com/everruns/fetchkit/pull/7)) by @chaliy +* docs: clarify test classification in AGENTS.md ([#6](https://github.com/everruns/fetchkit/pull/6)) by @chaliy +* docs: add cloud agent env and complete AGENTS.md placeholders ([#5](https://github.com/everruns/fetchkit/pull/5)) by @chaliy +* refactor: rename project from webfetch to fetchkit ([#4](https://github.com/everruns/fetchkit/pull/4)) by @chaliy +* docs: add comprehensive README with installation and usage guide ([#3](https://github.com/everruns/fetchkit/pull/3)) by @chaliy +* feat: implement webfetch library, CLI, MCP server, and Python bindings ([#1](https://github.com/everruns/fetchkit/pull/1)) by @chaliy +* feat: add initial webfetch spec and guidance by @chaliy + +**Full Changelog**: https://github.com/everruns/fetchkit/commits/v0.1.0 + +[Unreleased]: https://github.com/everruns/fetchkit/compare/v0.1.0...HEAD +[0.1.0]: https://github.com/everruns/fetchkit/releases/tag/v0.1.0 diff --git a/Cargo.toml b/Cargo.toml index c5b3c7d..53f14c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,9 @@ edition = "2021" license = "MIT" authors = ["Everruns"] repository = "https://github.com/everruns/fetchkit" -description = "AI-friendly fetchkit tool, CLI, MCP server, and library" +description = "AI-friendly web content fetching and HTML-to-Markdown conversion library" +keywords = ["fetch", "web", "markdown", "llm", "ai"] +categories = ["web-programming", "text-processing"] [workspace.dependencies] # Async runtime diff --git a/crates/fetchkit-cli/Cargo.toml b/crates/fetchkit-cli/Cargo.toml index 02ff521..80a42c6 100644 --- a/crates/fetchkit-cli/Cargo.toml +++ b/crates/fetchkit-cli/Cargo.toml @@ -5,14 +5,17 @@ edition.workspace = true license.workspace = true authors.workspace = true repository.workspace = true -description = "CLI for the FetchKit tool" +description = "Command line interface for FetchKit web content fetching tool" +keywords.workspace = true +categories.workspace = true +readme = "../../README.md" [[bin]] name = "fetchkit" path = "src/main.rs" [dependencies] -fetchkit = { version = "0.1.0", path = "../fetchkit" } +fetchkit = { path = "../fetchkit", version = "0.1.0" } tokio = { workspace = true } clap = { workspace = true } serde = { workspace = true } diff --git a/crates/fetchkit-python/Cargo.toml b/crates/fetchkit-python/Cargo.toml index d82d812..5895b9a 100644 --- a/crates/fetchkit-python/Cargo.toml +++ b/crates/fetchkit-python/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true authors.workspace = true repository.workspace = true description = "Python bindings for the FetchKit library" +publish = false [lib] name = "fetchkit_py" diff --git a/crates/fetchkit/Cargo.toml b/crates/fetchkit/Cargo.toml index 3e69044..e4dbd23 100644 --- a/crates/fetchkit/Cargo.toml +++ b/crates/fetchkit/Cargo.toml @@ -5,7 +5,10 @@ edition.workspace = true license.workspace = true authors.workspace = true repository.workspace = true -description = "AI-friendly fetchkit library for fetching and converting web content" +description.workspace = true +keywords.workspace = true +categories.workspace = true +readme = "../../README.md" [dependencies] tokio = { workspace = true } diff --git a/docs/release-process.md b/docs/release-process.md index 0dfc5ad..814b208 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -18,9 +18,9 @@ fetchkit follows [Semantic Versioning](https://semver.org/): ``` ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ -│ Human asks │ │ Agent creates │ │ GitHub │ │ crates.io │ -│ "release v0.2" │────>│ release PR │────>│ Release │────>│ Publish │ -│ │ │ │ │ (automatic) │ │ (automatic) │ +│ Human asks │ │ Agent creates │ │ GitHub │ │ crates.io │ +│ "release v0.2" │────>│ release PR │────>│ Release │────>│ Publish │ +│ │ │ │ │ (automatic) │ │ (automatic) │ └─────────────────┘ └─────────────────┘ └─────────────────┘ └─────────────────┘ ``` @@ -46,17 +46,20 @@ When asked to create a release, the agent: 2. **Update CHANGELOG.md** - Move items from `[Unreleased]` to new version section - Add release date: `## [X.Y.Z] - YYYY-MM-DD` - - Add breaking changes section if applicable (see format below) - - List commits in GitHub-style format with PR links and contributors + - Add `### Highlights` section with 2-5 key changes + - Add `### Breaking Changes` section if applicable (see format below) + - List commits in `### What's Changed` using `*` bullets with PR links and contributors + - Add `**Full Changelog**` comparison link at bottom of version section - Update comparison links at bottom of file 3. **Update Cargo.toml** - Set `version = "X.Y.Z"` in workspace + - Update `fetchkit-cli` dependency version on `fetchkit` to match 4. **Run verification** - `cargo fmt --check` - - `cargo clippy` - - `cargo test` + - `cargo clippy --workspace --all-targets -- -D warnings` + - `cargo test --workspace` 5. **Commit and push** - Commit message: `chore(release): prepare vX.Y.Z` @@ -70,12 +73,17 @@ When asked to create a release, the agent: **On merge to main** (release.yml): - Detects commit message `chore(release): prepare vX.Y.Z` +- Also supports manual trigger via `workflow_dispatch` - Extracts release notes from CHANGELOG.md -- Creates GitHub Release with tag `vX.Y.Z` +- Creates GitHub Release with tag `vX.Y.Z` using `softprops/action-gh-release` +- Triggers the publish workflow **On GitHub Release created** (publish.yml): -- Runs verification (fmt, clippy, tests) -- Publishes `fetchkit` and `fetchkit-cli` to crates.io +- Verifies tag version matches Cargo.toml version +- Publishes `fetchkit` to crates.io +- Waits 30s for crates.io index update +- Publishes `fetchkit-cli` to crates.io +- Also supports manual trigger via `workflow_dispatch` ## Pre-Release Checklist @@ -96,6 +104,11 @@ The changelog follows [Keep a Changelog](https://keepachangelog.com/) with GitHu ```markdown ## [X.Y.Z] - YYYY-MM-DD +### Highlights + +- Key change 1 +- Key change 2 + ### Breaking Changes - **Short description**: Detailed explanation of what changed and migration steps. @@ -104,8 +117,10 @@ The changelog follows [Keep a Changelog](https://keepachangelog.com/) with GitHu ### What's Changed -- Commit message ([#PR](https://github.com/everruns/fetchkit/pull/PR)) by @contributor -- Another commit ([#PR](https://github.com/everruns/fetchkit/pull/PR)) by @contributor +* type(scope): commit message ([#PR](https://github.com/everruns/fetchkit/pull/PR)) by @contributor +* type(scope): another commit ([#PR](https://github.com/everruns/fetchkit/pull/PR)) by @contributor + +**Full Changelog**: https://github.com/everruns/fetchkit/compare/vPREV...vX.Y.Z ``` ### Generating Commit List @@ -118,7 +133,7 @@ git log --oneline | grep -v -E "^.{7} (chore|ci|bench)" Format each commit as: ``` -- ([#](https://github.com/everruns/fetchkit/pull/)) by @ +* ([#](https://github.com/everruns/fetchkit/pull/)) by @ ``` ### Breaking Changes Section @@ -129,30 +144,29 @@ Include when the release has breaking changes (typically MINOR or MAJOR versions 2. **Migration guide** showing before/after 3. **Code examples** if helpful -Example: -```markdown -### Breaking Changes - -- **Renamed crate from webfetch to fetchkit**: All imports need to be updated. - - `webfetch::fetch` → `fetchkit::fetch` - - CLI binary: `webfetch` → `fetchkit` -``` - ## Workflows ### release.yml -- **Trigger**: Push to `main` with commit message starting with `chore(release): prepare v` -- **Actions**: Creates GitHub Release with tag and release notes +- **Trigger**: Push to `main` with commit message starting with `chore(release): prepare v`, or manual `workflow_dispatch` +- **Actions**: Creates GitHub Release with tag and release notes, triggers publish workflow - **File**: `.github/workflows/release.yml` ### publish.yml -- **Trigger**: GitHub Release published -- **Actions**: Verifies and publishes to crates.io +- **Trigger**: GitHub Release published, or manual `workflow_dispatch` +- **Actions**: Verifies version and publishes to crates.io - **File**: `.github/workflows/publish.yml` - **Secret required**: `CARGO_REGISTRY_TOKEN` +## Package Distribution + +| Package | Registry | Notes | +|---------|----------|-------| +| `fetchkit` | crates.io | Core library | +| `fetchkit-cli` | crates.io | CLI binary (`cargo install fetchkit-cli`) | +| `fetchkit-python` | PyPI (future) | Python bindings, not published to crates.io | + ## Example Conversation ``` @@ -185,9 +199,8 @@ Each release includes: - **GitHub Release**: Tag, release notes, source archives - **crates.io**: Published crates for `cargo install fetchkit-cli` -Note: `fetchkit-python` is not published to crates.io (uses PyPI distribution instead). - Future considerations: - Pre-built binaries (Linux, macOS, Windows) - Docker images - Homebrew formula +- Python wheels on PyPI