Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 17 additions & 64 deletions .github/workflows/publish.yml
Original file line number Diff line number Diff line change
@@ -1,100 +1,53 @@
name: Publish to crates.io
name: Publish

on:
release:
types: [published]
workflow_dispatch:
inputs:
dry_run:
description: 'Dry run (no actual publish)'
required: false
default: 'true'
type: boolean
crate:
description: 'Crate to publish (all, fetchkit, fetchkit-cli)'
required: false
default: 'all'
type: choice
options:
- all
- fetchkit
- fetchkit-cli

permissions:
contents: read

env:
CARGO_TERM_COLOR: always
RUST_BACKTRACE: 1

jobs:
test:
name: Test before publish
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- name: Run tests
run: cargo test --workspace
- name: Check formatting
run: cargo fmt --all -- --check
- name: Clippy
run: cargo clippy --workspace --all-targets -- -D warnings

publish-fetchkit:
name: Publish fetchkit
needs: test
runs-on: ubuntu-latest
if: >-
github.event_name == 'release' ||
(github.event_name == 'workflow_dispatch' &&
(github.event.inputs.crate == 'all' || github.event.inputs.crate == 'fetchkit'))
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2

- name: Verify crate can be packaged
run: cargo package -p fetchkit --allow-dirty

- name: Publish fetchkit (dry run)
if: github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true'
run: cargo publish -p fetchkit --dry-run

- name: Publish fetchkit
if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'false')
- name: Verify version matches tag
run: |
TAG_VERSION="${{ github.event.release.tag_name }}"
if [ -n "$TAG_VERSION" ]; then
TAG_VERSION="${TAG_VERSION#v}"
CARGO_VERSION=$(grep '^version' Cargo.toml | head -1 | sed 's/.*"\(.*\)".*/\1/')
if [ "$TAG_VERSION" != "$CARGO_VERSION" ]; then
echo "Error: Tag version does not match Cargo.toml version"
exit 1
fi
fi

- name: Publish fetchkit to crates.io
run: cargo publish -p fetchkit
env:
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}

publish-fetchkit-cli:
name: Publish fetchkit-cli
needs: [test, publish-fetchkit]
runs-on: ubuntu-latest
if: >-
github.event_name == 'release' ||
(github.event_name == 'workflow_dispatch' &&
(github.event.inputs.crate == 'all' || github.event.inputs.crate == 'fetchkit-cli'))
needs: publish-fetchkit
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2

# Wait for crates.io index to update after fetchkit publish
- name: Wait for crates.io index
if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'false')
- name: Wait for crates.io index update
run: sleep 30

- name: Verify crate can be packaged
run: cargo package -p fetchkit-cli --allow-dirty

- name: Publish fetchkit-cli (dry run)
if: github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true'
run: cargo publish -p fetchkit-cli --dry-run

- name: Publish fetchkit-cli
if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'false')
- name: Publish fetchkit-cli to crates.io
run: cargo publish -p fetchkit-cli
env:
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
86 changes: 31 additions & 55 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,87 +4,63 @@ on:
push:
branches:
- main
workflow_dispatch:

permissions:
contents: write
actions: write

jobs:
release:
name: Create Release
runs-on: ubuntu-latest
# Only run if commit message matches release pattern
if: ${{ startsWith(github.event.head_commit.message, 'chore(release): prepare v') }}

permissions:
contents: write
if: "${{ github.event_name == 'workflow_dispatch' || startsWith(github.event.head_commit.message, 'chore(release): prepare v') }}"

steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Extract version from commit message
- name: Extract version
id: version
run: |
# Extract version from commit message like "chore(release): prepare v0.1.0"
VERSION=$(echo "${{ github.event.head_commit.message }}" | grep -oP 'prepare v\K[0-9]+\.[0-9]+\.[0-9]+')
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
VERSION=$(grep '^version' Cargo.toml | head -1 | sed 's/.*"\(.*\)".*/\1/')
else
COMMIT_MSG="${{ github.event.head_commit.message }}"
VERSION=$(echo "$COMMIT_MSG" | sed -n 's/.*prepare v\([0-9]*\.[0-9]*\.[0-9]*\).*/\1/p')
fi
if [ -z "$VERSION" ]; then
echo "::error::Could not extract version from commit message"
echo "Error: Could not determine version"
exit 1
fi
echo "version=$VERSION" >> $GITHUB_OUTPUT
echo "tag=v$VERSION" >> $GITHUB_OUTPUT
echo "Extracted version: $VERSION"

- name: Check if tag already exists
id: check_tag
run: |
if git rev-parse "v${{ steps.version.outputs.version }}" >/dev/null 2>&1; then
echo "::error::Tag v${{ steps.version.outputs.version }} already exists"
exit 1
fi
echo "Tag does not exist, proceeding..."

- name: Verify Cargo.toml version matches
- name: Verify version matches Cargo.toml
run: |
CARGO_VERSION=$(grep '^version = ' Cargo.toml | head -1 | sed 's/version = "\(.*\)"/\1/')
if [ "$CARGO_VERSION" != "${{ steps.version.outputs.version }}" ]; then
echo "::error::Cargo.toml version ($CARGO_VERSION) does not match release version (${{ steps.version.outputs.version }})"
CARGO_VERSION=$(grep '^version' Cargo.toml | head -1 | sed 's/.*"\(.*\)".*/\1/')
if [ "${{ steps.version.outputs.version }}" != "$CARGO_VERSION" ]; then
echo "Error: Commit version does not match Cargo.toml version"
exit 1
fi
echo "Cargo.toml version matches: $CARGO_VERSION"

- name: Extract release notes from CHANGELOG.md
- name: Extract release notes from CHANGELOG
id: changelog
run: |
VERSION="${{ steps.version.outputs.version }}"

# Extract the section for this version from CHANGELOG.md
# Matches from "## [X.Y.Z]" until the next "## [" or end of significant content
NOTES=$(awk -v ver="$VERSION" '
/^## \[/ {
if (found) exit
if (index($0, "[" ver "]")) found=1
next
}
found && /^## \[/ { exit }
found { print }
' CHANGELOG.md)

if [ -z "$NOTES" ]; then
echo "::warning::No changelog entry found for version $VERSION"
NOTES="Release v$VERSION"
fi

# Write to file to preserve formatting
NOTES=$(awk "/^## \[$VERSION\]/{found=1; next} /^## \[/{if(found) exit} found{print}" CHANGELOG.md)
echo "$NOTES" > release_notes.md
echo "Release notes extracted:"
cat release_notes.md

- name: Create GitHub Release
uses: softprops/action-gh-release@v2
with:
tag_name: ${{ steps.version.outputs.tag }}
name: Release ${{ steps.version.outputs.tag }}
body_path: release_notes.md
draft: false
prerelease: false

- name: Trigger publish workflow
run: gh workflow run publish.yml

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Include release ref when triggering publish workflow

release.yml dispatches publish with gh workflow run publish.yml but does not pass any release tag/ref context, so publish.yml runs as workflow_dispatch and github.event.release.tag_name is empty, which skips the version guard in Verify version matches tag. That means publish is no longer cryptographically tied to the release artifact and can publish whatever revision/version the dispatch resolves to (for example on reruns or branch movement), defeating the intended tag-version safety check.

Useful? React with 👍 / 👎.

env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh release create "v${{ steps.version.outputs.version }}" \
--title "v${{ steps.version.outputs.version }}" \
--notes-file release_notes.md \
--target "${{ github.sha }}"

echo "Created release v${{ steps.version.outputs.version }}"
13 changes: 7 additions & 6 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -98,19 +98,20 @@ See `docs/release-process.md` for full release process documentation.

Quick summary:
1. Human asks agent: "Create release v0.2.0"
2. Agent updates CHANGELOG.md, Cargo.toml version, creates PR
2. Agent updates CHANGELOG.md (with Highlights + What's Changed), Cargo.toml version, creates PR
3. Human reviews and merges PR to main
4. CI creates GitHub Release (release.yml)
5. CI publishes to crates.io (publish.yml)
4. CI creates GitHub Release via `softprops/action-gh-release` (release.yml)
5. release.yml triggers publish.yml
6. CI publishes `fetchkit` then `fetchkit-cli` to crates.io (publish.yml)

Workflows:
- `.github/workflows/release.yml` - Creates GitHub Release on merge
- `.github/workflows/publish.yml` - Publishes to crates.io on GitHub Release
- `.github/workflows/release.yml` - Creates GitHub Release on merge or manual dispatch
- `.github/workflows/publish.yml` - Publishes to crates.io on GitHub Release or manual dispatch

Requirements:
- `CARGO_REGISTRY_TOKEN` secret must be configured in repo settings

Note: `fetchkit-python` is not published to crates.io (uses PyPI distribution instead).
Note: `fetchkit-python` is not published to crates.io (`publish = false`). Uses PyPI distribution instead.

### Cloud Agent environments

Expand Down
34 changes: 23 additions & 11 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

## [0.1.0] - 2026-02-12

### Highlights

- AI-friendly web content fetching with HTML-to-Markdown and HTML-to-Text conversion
- CLI and MCP server for AI tool integration
- Pluggable fetcher system for URL-specific handling
- Python bindings via PyO3

### What's Changed

- feat: add pluggable fetcher system for URL-specific handling ([#9](https://github.com/everruns/fetchkit/pull/9)) by @chaliy
- docs: add LangChain example for MCP integration ([#8](https://github.com/everruns/fetchkit/pull/8)) by @chaliy
- refactor(cli): unified md-first output format ([#7](https://github.com/everruns/fetchkit/pull/7)) by @chaliy
- docs: clarify test classification in AGENTS.md ([#6](https://github.com/everruns/fetchkit/pull/6)) by @chaliy
- docs: add cloud agent env and complete AGENTS.md placeholders ([#5](https://github.com/everruns/fetchkit/pull/5)) by @chaliy
- refactor: rename project from webfetch to fetchkit ([#4](https://github.com/everruns/fetchkit/pull/4)) by @chaliy
- docs: add comprehensive README with installation and usage guide ([#3](https://github.com/everruns/fetchkit/pull/3)) by @chaliy
- feat: implement webfetch library, CLI, MCP server, and Python bindings ([#1](https://github.com/everruns/fetchkit/pull/1)) by @chaliy
- feat: add initial webfetch spec and guidance by @chaliy

[Unreleased]: https://github.com/everruns/fetchkit/compare/HEAD...HEAD
* feat: add pluggable fetcher system for URL-specific handling ([#9](https://github.com/everruns/fetchkit/pull/9)) by @chaliy
* docs: add LangChain example for MCP integration ([#8](https://github.com/everruns/fetchkit/pull/8)) by @chaliy
* refactor(cli): unified md-first output format ([#7](https://github.com/everruns/fetchkit/pull/7)) by @chaliy
* docs: clarify test classification in AGENTS.md ([#6](https://github.com/everruns/fetchkit/pull/6)) by @chaliy
* docs: add cloud agent env and complete AGENTS.md placeholders ([#5](https://github.com/everruns/fetchkit/pull/5)) by @chaliy
* refactor: rename project from webfetch to fetchkit ([#4](https://github.com/everruns/fetchkit/pull/4)) by @chaliy
* docs: add comprehensive README with installation and usage guide ([#3](https://github.com/everruns/fetchkit/pull/3)) by @chaliy
* feat: implement webfetch library, CLI, MCP server, and Python bindings ([#1](https://github.com/everruns/fetchkit/pull/1)) by @chaliy
* feat: add initial webfetch spec and guidance by @chaliy

**Full Changelog**: https://github.com/everruns/fetchkit/commits/v0.1.0

[Unreleased]: https://github.com/everruns/fetchkit/compare/v0.1.0...HEAD
[0.1.0]: https://github.com/everruns/fetchkit/releases/tag/v0.1.0
4 changes: 3 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ edition = "2021"
license = "MIT"
authors = ["Everruns"]
repository = "https://github.com/everruns/fetchkit"
description = "AI-friendly fetchkit tool, CLI, MCP server, and library"
description = "AI-friendly web content fetching and HTML-to-Markdown conversion library"
keywords = ["fetch", "web", "markdown", "llm", "ai"]
categories = ["web-programming", "text-processing"]

[workspace.dependencies]
# Async runtime
Expand Down
7 changes: 5 additions & 2 deletions crates/fetchkit-cli/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,17 @@ edition.workspace = true
license.workspace = true
authors.workspace = true
repository.workspace = true
description = "CLI for the FetchKit tool"
description = "Command line interface for FetchKit web content fetching tool"
keywords.workspace = true
categories.workspace = true
readme = "../../README.md"

[[bin]]
name = "fetchkit"
path = "src/main.rs"

[dependencies]
fetchkit = { version = "0.1.0", path = "../fetchkit" }
fetchkit = { path = "../fetchkit", version = "0.1.0" }
tokio = { workspace = true }
clap = { workspace = true }
serde = { workspace = true }
Expand Down
1 change: 1 addition & 0 deletions crates/fetchkit-python/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ license.workspace = true
authors.workspace = true
repository.workspace = true
description = "Python bindings for the FetchKit library"
publish = false

[lib]
name = "fetchkit_py"
Expand Down
5 changes: 4 additions & 1 deletion crates/fetchkit/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,10 @@ edition.workspace = true
license.workspace = true
authors.workspace = true
repository.workspace = true
description = "AI-friendly fetchkit library for fetching and converting web content"
description.workspace = true
keywords.workspace = true
categories.workspace = true
readme = "../../README.md"

[dependencies]
tokio = { workspace = true }
Expand Down
Loading