Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
0d9a80e
refactor!: rework proxy to use multistore on cloudflare workers runtime
alukach Mar 17, 2026
e181079
chore: rm claude litter
alukach Mar 17, 2026
2e219e1
chore: add routes
alukach Mar 17, 2026
ec9b242
Fix name and prefix
alukach Mar 18, 2026
7026cf4
fix: strip authentication headers
alukach Mar 18, 2026
8d32025
fix: correct prefix
alukach Mar 18, 2026
cc0430d
chore: improve logging
alukach Mar 18, 2026
6b5bde1
feat: enable azure backend
alukach Mar 18, 2026
d3ade44
ci: increase testing
alukach Mar 18, 2026
9538d96
chore: update docs
alukach Mar 18, 2026
f32dbfb
chore: bump multistore version
alukach Mar 18, 2026
f2cf3ed
chore: set app log level
alukach Mar 18, 2026
836b3d2
chore: log 404s as warns
alukach Mar 18, 2026
7519c4f
chore: cargo fmt
alukach Mar 18, 2026
8b59ff2
feat: add Cache API wrapper with per-datatype TTLs
alukach Mar 18, 2026
00ac28e
feat: use cached API calls in registry
alukach Mar 18, 2026
16ca184
feat: add cache
alukach Mar 18, 2026
9b56c86
chore: bump version
alukach Mar 18, 2026
421bdcf
refactor: simplify proxy using multistore-cf-workers and multistore-p…
alukach Mar 18, 2026
443ae49
fix: cache key based on URL
alukach Mar 18, 2026
a13ab7b
chore: update routes, add staging
alukach Mar 18, 2026
3f84793
fix: handle percent encoded paths
alukach Mar 18, 2026
1eed806
refactor: extract routing into module with native-target tests
alukach Mar 18, 2026
390443e
refactor: prefer explicit imports
alukach Mar 18, 2026
009191e
chore: pin version to multistore repo
alukach Mar 18, 2026
fed9a2b
refactor: use multistore responses
alukach Mar 19, 2026
27de354
feat: make use of pagination
alukach Mar 19, 2026
6828d31
fix: use XML errors
alukach Mar 19, 2026
1c39c84
fix: encode prefix
alukach Mar 19, 2026
33d4727
chore: add observability settings
alukach Mar 19, 2026
34291e8
fix: match s3 encoding behavior
alukach Mar 20, 2026
18f6d81
chore: bump multistore version
alukach Mar 20, 2026
d4150b0
feat: support authenticated requests to API
alukach Mar 20, 2026
90b2b21
chore: cargo fmt
alukach Mar 20, 2026
28934e0
feat: improve observability
alukach Mar 20, 2026
a5dda34
docs: build out readme
alukach Mar 20, 2026
0a44d6b
chore: rm alpha.data.source.coop route
alukach Mar 23, 2026
e009fb9
chore: cargo audit
alukach Mar 23, 2026
fe91da7
feat: add full errors in responses in debug mode
alukach Mar 23, 2026
0a5dec7
ci: add deployment tooling
alukach Mar 24, 2026
4bfe4b7
fix deployment
alukach Mar 24, 2026
820672a
ci: fix preview
alukach Mar 24, 2026
cd82e51
ci: update variables
alukach Mar 24, 2026
d46a77d
ci: use prebuilt cargo-audit
alukach Mar 24, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 0 additions & 53 deletions .dockerignore

This file was deleted.

19 changes: 0 additions & 19 deletions .github/workflows/ci.yaml

This file was deleted.

110 changes: 110 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
name: CI

on:
push:
branches: [main]
pull_request:
branches: [main]

env:
CARGO_TERM_COLOR: always

jobs:
fmt:
name: Format
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
with:
components: rustfmt
- run: cargo fmt --all -- --check

clippy:
name: Clippy
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
with:
components: clippy
targets: wasm32-unknown-unknown
- uses: Swatinem/rust-cache@v2
- run: cargo clippy --target wasm32-unknown-unknown -- -D warnings

test:
name: Unit Tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- run: cargo test

check:
name: Check
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
with:
targets: wasm32-unknown-unknown
- uses: Swatinem/rust-cache@v2
- run: cargo check --target wasm32-unknown-unknown

build:
name: Build Worker
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
with:
targets: wasm32-unknown-unknown
- uses: Swatinem/rust-cache@v2
- name: Install worker-build
run: cargo install worker-build@0.7.5
- name: Build WASM worker
run: worker-build --release

integration:
name: Integration Tests
runs-on: ubuntu-latest
needs: [build]
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
with:
targets: wasm32-unknown-unknown
- uses: Swatinem/rust-cache@v2
- uses: actions/setup-node@v4
with:
node-version: "22"
- uses: astral-sh/setup-uv@v5
- name: Install wrangler
run: npm install -g wrangler@3
- name: Install worker-build
run: cargo install worker-build@0.7.5
- name: Start wrangler dev
run: |
wrangler dev --port 8787 &
for i in $(seq 1 60); do
if curl -s --max-time 2 http://localhost:8787/ > /dev/null 2>&1; then
echo "Server ready after ${i}s"
break
fi
sleep 1
done
- name: Run integration tests
run: uvx --with requests pytest tests/ -v

audit:
name: Security Audit
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- name: Install cargo-audit
uses: taiki-e/install-action@v2
with:
tool: cargo-audit
- run: cargo audit
4 changes: 2 additions & 2 deletions .github/workflows/conventional-commits.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ jobs:
- uses: actions/create-github-app-token@5d869da34e18e7287c1daad50e0b8ea0f506ce69 # v 1.11.0
id: app-token
with:
app-id: ${{ vars.APP_ID }}
private-key: ${{ secrets.PRIVATE_KEY }}
app-id: ${{ vars.SC_RELEASE_BOT_ID }}
private-key: ${{ secrets.SC_RELEASE_BOT_PRIVATE_KEY }}

- name: PR Conventional Commit Validation
uses: ytanikin/pr-conventional-commits@1.4.0
Expand Down
79 changes: 79 additions & 0 deletions .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
name: Deploy

on:
workflow_call:
inputs:
worker_name:
description: "Cloudflare Worker name override (cannot be used with wrangler_env)"
required: false
type: string
default: ""
wrangler_env:
description: "Wrangler environment (e.g. staging). Cannot be used with worker_name."
required: false
type: string
default: ""
wrangler_config:
description: "Wrangler config file (default: wrangler.toml)"
required: false
type: string
default: "wrangler.toml"
var_overrides:
description: "Additional --var flags (e.g. 'LOG_LEVEL:DEBUG SOURCE_API_URL:https://staging.source.coop')"
required: false
type: string
default: ""
environment:
description: "GitHub environment name (preview, staging, production)"
required: true
type: string
outputs:
deploy_url:
description: "The deployed worker URL"
value: ${{ jobs.deploy.outputs.deploy_url }}
secrets:
CLOUDFLARE_API_TOKEN:
required: true
CLOUDFLARE_ACCOUNT_ID:
required: true

jobs:
deploy:
name: Deploy
runs-on: ubuntu-latest
environment: ${{ inputs.environment }}
outputs:
deploy_url: ${{ steps.url.outputs.deploy_url }}
env:
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
with:
targets: wasm32-unknown-unknown
- uses: Swatinem/rust-cache@v2
- uses: actions/setup-node@v4
with:
node-version: "22"

- name: Deploy worker
run: |
EXTRA_ARGS=""
if [ -n "${{ inputs.wrangler_env }}" ]; then
EXTRA_ARGS="--env ${{ inputs.wrangler_env }}"
fi
if [ -n "${{ inputs.worker_name }}" ]; then
EXTRA_ARGS="--name ${{ inputs.worker_name }}"
fi
for var in ${{ inputs.var_overrides }}; do
EXTRA_ARGS="$EXTRA_ARGS --var $var"
done
npx wrangler@3 deploy --config ${{ inputs.wrangler_config }} $EXTRA_ARGS

- name: Get deploy URL
id: url
run: |
WORKER="${{ inputs.worker_name || 'source-data-proxy' }}"
URL="https://${WORKER}.${{ vars.CLOUDFLARE_WORKERS_SUBDOMAIN }}.workers.dev"
echo "deploy_url=$URL" >> "$GITHUB_OUTPUT"
4 changes: 2 additions & 2 deletions .github/workflows/please-release.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ jobs:
- uses: actions/create-github-app-token@5d869da34e18e7287c1daad50e0b8ea0f506ce69 # v 1.11.0
id: app-token
with:
app-id: ${{ vars.APP_ID }}
private-key: ${{ secrets.PRIVATE_KEY }}
app-id: ${{ vars.SC_RELEASE_BOT_ID }}
private-key: ${{ secrets.SC_RELEASE_BOT_PRIVATE_KEY }}

- uses: googleapis/release-please-action@078b9b8dda7799db29972b379561ff8e19b08e82 # v4
with:
Expand Down
62 changes: 62 additions & 0 deletions .github/workflows/preview.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
name: Preview

on:
pull_request:
types: [opened, synchronize, reopened, closed]

concurrency:
group: preview-${{ github.event.pull_request.number }}
cancel-in-progress: true

jobs:
deploy:
name: Deploy & Test
if: github.event.action != 'closed'
uses: ./.github/workflows/deploy.yml
with:
worker_name: source-data-proxy-pr-${{ github.event.pull_request.number }}
wrangler_config: wrangler.preview.toml
var_overrides: "LOG_LEVEL:DEBUG SOURCE_API_URL:https://staging.source.coop"
environment: preview
secrets:
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}

comment:
name: PR Comment
needs: deploy
if: github.event.action != 'closed'
runs-on: ubuntu-latest
permissions:
pull-requests: write
steps:
- name: Find Comment
uses: peter-evans/find-comment@v4
id: find-comment
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: "github-actions[bot]"
body-includes: Latest commit deployed to

- name: Create or update comment
uses: peter-evans/create-or-update-comment@v5
with:
comment-id: ${{ steps.find-comment.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body: |
🚀 Latest commit deployed to ${{ needs.deploy.outputs.deploy_url }}

* Date: `${{ github.event.pull_request.updated_at }}`
* Commit: ${{ github.sha }}
edit-mode: replace

cleanup:
name: Cleanup Preview
if: github.event.action == 'closed'
runs-on: ubuntu-latest
env:
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
steps:
- name: Delete preview worker
run: npx wrangler@3 delete --name "source-data-proxy-pr-${{ github.event.pull_request.number }}" --force
46 changes: 0 additions & 46 deletions .github/workflows/prod-deploy.yaml

This file was deleted.

Loading
Loading