Skip to content

Commit 4168e45

Browse files
ClemDoumClemDoum
authored
feature: HTTP task service (#66)
1 parent 27eb03e commit 4168e45

18 files changed

Lines changed: 1491 additions & 10 deletions

‎.github/workflows/publish-common.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,18 @@ on:
66
- icij-common-*
77

88
jobs:
9+
create-release:
10+
runs-on: ubuntu-latest
11+
steps:
12+
- uses: actions/checkout@v4
13+
- name: Create GH release
14+
run: gh release create "$tag" --generate-notes
15+
env:
16+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
17+
tag: ${{ github.ref_name }}
18+
919
publish-icij-common:
20+
needs: [create-release]
1021
runs-on: ubuntu-latest
1122
environment:
1223
name: pypi-icij-common

‎.github/workflows/publish-worker.yml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,18 @@ on:
66
- icij-worker-*
77

88
jobs:
9+
create-release:
10+
runs-on: ubuntu-latest
11+
steps:
12+
- uses: actions/checkout@v4
13+
- name: Create GH release
14+
run: gh release create "$tag" --generate-notes
15+
env:
16+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
17+
tag: ${{ github.ref_name }}
18+
919
publish-icij-worker:
20+
needs: [create-release]
1021
runs-on: ubuntu-latest
1122
environment:
1223
name: pypi-icij-worker
@@ -26,6 +37,54 @@ jobs:
2637
uv build
2738
uv publish --trusted-publishing always
2839
40+
publish-http-service-docker:
41+
needs: [ create-release ]
42+
runs-on: ubuntu-latest
43+
steps:
44+
- name: Determine DBMate architecture
45+
id: arch
46+
run: |
47+
DBMATE_ARCH=$(arch || uname -m)
48+
if [[ "$DBMATE_ARCH" == "x86_64" || "$DBMATE_ARCH" == "amd64" ]]; then
49+
DBMATE_ARCH="amd64"
50+
elif [[ "$DBMATE_ARCH" == "aarch64" || "$DBMATE_ARCH" == "arm64" ]]; then
51+
DBMATE_ARCH="arm64"
52+
elif [[ "$DBMATE_ARCH" == "i386" ]]; then
53+
DBMATE_ARCH="386"
54+
else
55+
echo "Unsupported architecture: $DBMATE_ARCH" >&2
56+
exit 1
57+
fi
58+
echo "DBMATE_ARCH=$DBMATE_ARCH" >> $GITHUB_ENV
59+
60+
- name: Docker meta
61+
id: meta
62+
uses: docker/metadata-action@v5
63+
with:
64+
images: icij/task-service
65+
66+
- name: Set up QEMU
67+
uses: docker/setup-qemu-action@v3
68+
69+
- name: Set up Docker Buildx
70+
uses: docker/setup-buildx-action@v3
71+
72+
- name: Login to Docker Hub
73+
uses: docker/login-action@v3
74+
with:
75+
username: ${{ secrets.DOCKERHUB_USERNAME }}
76+
password: ${{ secrets.DOCKERHUB_TOKEN }}
77+
78+
- name: Build and push the service image
79+
uses: docker/build-push-action@v6
80+
with:
81+
context: icij-worker
82+
target: http-service
83+
platforms: linux/amd64,linux/arm64
84+
push: true
85+
tags: ${{ steps.meta.outputs.tags }}
86+
labels: ${{ steps.meta.outputs.labels }}
87+
2988
concurrency:
3089
group: ${{ github.workflow }}-${{ github.ref }}
3190
cancel-in-progress: false

‎icij-worker/Dockerfile‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# syntax=docker/dockerfile:1.14.0
2+
FROM python:3.11-slim-bullseye AS python-base
3+
4+
ENV HOME=/home/user
5+
WORKDIR $HOME
6+
RUN apt-get update && apt-get install -y curl
7+
8+
RUN curl -LsSf https://astral.sh/uv/0.6.7/install.sh | sh
9+
ENV PATH="$HOME/.local/bin:$PATH"
10+
ENV UV_LINK_MODE=copy
11+
ENV UV_COMPILE_BYTECODE=1
12+
13+
##### HTTP serivce
14+
FROM python-base AS http-service
15+
16+
ARG dbmate_arch
17+
WORKDIR $HOME/src/app
18+
RUN curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v2.19.0/dbmate-linux-${dbmate_arch} \
19+
&& chmod +x /usr/local/bin/dbmate
20+
# Install deps first to optimize layer cache
21+
RUN --mount=type=cache,target=~/.cache/uv \
22+
--mount=type=bind,source=uv.lock,target=uv.lock \
23+
--mount=type=bind,source=pyproject.toml,target=pyproject.toml \
24+
uv sync -v --frozen --no-editable --no-sources --no-install-project --extra http --extra amqp --extra postgres
25+
# Then copy code
26+
ADD uv.lock pyproject.toml README.md ./
27+
ADD icij_worker ./icij_worker/
28+
# Then install service
29+
RUN uv sync -v --frozen --no-editable --no-sources --extra http --extra amqp --extra postgres
30+
RUN uv pip list
31+
RUN rm -rf ~/.cache/pip $(uv cache dir)
32+
33+
ENTRYPOINT ["uv", "run", "--frozen", "--no-editable", "--no-sources", "icij-http-server"]

‎icij-worker/docker-compose.yml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
version: '3.7'
2+
3+
x-tm-amqp-config-variables: &tm-amqp-config
4+
TASK_MANAGER__RABBITMQ_HOST: rabbitmq
5+
6+
x-postgres-storage-config: &tm-postgres-storage-config
7+
TASK_MANAGER__BACKEND: amqp
8+
# Change this to a FSKeyValueStorageConfig if you don't want to use postgres
9+
TASK_MANAGER__STORAGE__HOST: postgres
10+
TASK_MANAGER__STORAGE__PORT: 5432
11+
TASK_MANAGER__STORAGE__PASSWORD: changeme
12+
13+
14+
services:
15+
rabbitmq:
16+
image: rabbitmq:3.12.0-management
17+
container_name: http-service-rabbitmq
18+
healthcheck:
19+
test: rabbitmq-diagnostics -q status
20+
interval: 5s
21+
timeout: 2s
22+
retries: 10
23+
start_period: 5s
24+
ports:
25+
- "5672:5672"
26+
- "15672:15672"
27+
28+
postgres:
29+
image: postgres
30+
container_name: http-service-postgres
31+
environment:
32+
POSTGRES_PASSWORD: changeme
33+
healthcheck:
34+
test: pg_isready
35+
interval: 2s
36+
timeout: 2s
37+
retries: 10
38+
start_period: 5s
39+
ports:
40+
- "5435:5432"
41+
42+
http-service:
43+
depends_on:
44+
rabbitmq:
45+
condition: service_healthy
46+
postgres:
47+
condition: service_healthy
48+
build:
49+
context: .
50+
target: http-service
51+
args:
52+
dbmate_arch: ${DBMATE_ARCH}
53+
environment:
54+
<<: [ *tm-amqp-config, *tm-postgres-storage-config ]
55+
PORT: 8000
56+
# Uncomment this and set it to your app path
57+
#TASK_MANAGER__APP_PATH: path.to.app_module.app_variable
58+
59+
healthcheck:
60+
test: curl -f http://localhost:8000/health
61+
interval: 5s
62+
timeout: 2s
63+
retries: 10
64+
start_period: 5s
65+
ports:
66+
- "8000:8000"

‎icij-worker/http_service‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#!/usr/bin/env bash
2+
3+
function _export_globals() {
4+
DBMATE_ARCH=$(dbmate_arch)
5+
export DBMATE_ARCH
6+
}
7+
8+
function _helpers() {
9+
function dbmate_arch() {
10+
local host_arch
11+
if command -v arch >/dev/null 2>&1; then
12+
host_arch=$(arch)
13+
else
14+
host_arch=$(uname -m)
15+
fi
16+
local dbmate_arch_
17+
if [ "$host_arch" == "x86_64" ] ||[ "$host_arch" == "amd64" ]; then
18+
dbmate_arch_="amd64"
19+
elif [ "$host_arch" == "aarch64" ] || [ "$host_arch" == "arm64" ]; then
20+
dbmate_arch_="arm64"
21+
elif [ "$host_arch" == "i386" ] ; then
22+
dbmate_arch_="386"
23+
else
24+
_exit_with_message "Unsupported architecture $host_arch"
25+
fi
26+
echo "$dbmate_arch_"
27+
}
28+
29+
}
30+
31+
function _main() {
32+
set -e
33+
function _exit_with_message() {
34+
echo "$1"
35+
exit "${2:-1}"
36+
}
37+
_helpers
38+
_export_globals
39+
docker compose "$@"
40+
}
41+
42+
_main "$@"

‎icij-worker/icij_worker/cli/__init__.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,13 @@
66

77
import icij_worker
88
from icij_common.logging_utils import setup_loggers
9+
from icij_worker.cli.tasks import task_app
10+
from icij_worker.cli.utils import AsyncTyper
911
from icij_worker.cli.workers import worker_app
1012

11-
cli_app = typer.Typer(context_settings={"help_option_names": ["-h", "--help"]})
13+
cli_app = AsyncTyper(context_settings={"help_option_names": ["-h", "--help"]})
1214
cli_app.add_typer(worker_app)
15+
cli_app.add_typer(task_app)
1316

1417

1518
def version_callback(value: bool):

0 commit comments

Comments
 (0)