Skip to content

Commit cac38cb

Browse files
rostilosrostilos
authored
Merge pull request #158 from rostilos/1.5.3-rc
feat: Add support for repository access tokens in VCS connections
2 parents 25f9d64 + 8329c14 commit cac38cb

73 files changed

Lines changed: 2216 additions & 1244 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/deploy.yml‎

Lines changed: 14 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,8 @@
77
# 1. Checkout code (with submodules)
88
# 2. Build & test Java artifacts (Maven + JaCoCo)
99
# 3. Build all 5 Docker images
10-
# 4. Save images as a compressed tarball
11-
# 5. SCP tarball to production server
12-
# 6. SSH into server and run deployment script
10+
# 4. Push images to GHCR
11+
# 5. SSH into server and run deployment script
1312
#
1413
# Required GitHub Secrets:
1514
# DEPLOY_SSH_KEY — Private SSH key for env
@@ -19,6 +18,7 @@
1918
# ENV_INFERENCE_ORCHESTRATOR — Contents of inference-orchestrator/.env
2019
# ENV_RAG_PIPELINE — Contents of rag-pipeline/.env
2120
# ENV_WEB_FRONTEND — Contents of frontend/.env
21+
# GHCR_PAT — GitHub Personal Access Token for GHCR (optional, uses GITHUB_TOKEN if not set)
2222
###############################################################################
2323

2424
name: Deploy to Production
@@ -42,6 +42,7 @@ env:
4242

4343
permissions:
4444
contents: read
45+
packages: write
4546
checks: write
4647

4748
jobs:
@@ -68,11 +69,19 @@ jobs:
6869
- name: Set up Docker Buildx
6970
uses: docker/setup-buildx-action@v3
7071

71-
- name: Build (ci-build.sh)
72+
- name: Log in to the Container registry
73+
uses: docker/login-action@v3
74+
with:
75+
registry: ghcr.io
76+
username: ${{ github.actor }}
77+
password: ${{ secrets.GITHUB_TOKEN }}
78+
79+
- name: Build and Push (ci-build.sh)
7280
env:
7381
ENV_INFERENCE_ORCHESTRATOR: ${{ secrets.ENV_INFERENCE_ORCHESTRATOR }}
7482
ENV_RAG_PIPELINE: ${{ secrets.ENV_RAG_PIPELINE }}
7583
ENV_WEB_FRONTEND: ${{ secrets.ENV_WEB_FRONTEND }}
84+
GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }}
7685
run: |
7786
chmod +x deployment/ci/ci-build.sh
7887
deployment/ci/ci-build.sh
@@ -93,14 +102,6 @@ jobs:
93102
path: java-ecosystem/**/target/surefire-reports/
94103
retention-days: 7
95104

96-
- name: Upload artifact
97-
uses: actions/upload-artifact@v4
98-
with:
99-
name: codecrow-images
100-
path: build-output/codecrow-images.tar.zst
101-
retention-days: 7
102-
compression-level: 0
103-
104105
deploy:
105106
name: Deploy to Server
106107
runs-on: ubuntu-latest
@@ -113,13 +114,6 @@ jobs:
113114
- name: Checkout (for compose file + deploy script)
114115
uses: actions/checkout@v4
115116

116-
- name: Download artifact
117-
if: github.event.inputs.skip_build != 'true'
118-
uses: actions/download-artifact@v4
119-
with:
120-
name: codecrow-images
121-
path: build-output
122-
123117
- name: Setup SSH
124118
run: |
125119
mkdir -p ~/.ssh
@@ -136,18 +130,11 @@ jobs:
136130
deployment/ci/server-deploy.sh \
137131
${{ secrets.DEPLOY_USER }}@${{ secrets.DEPLOY_HOST }}:${{ env.DEPLOY_PATH }}/
138132
139-
- name: Upload Docker images tarball
140-
if: github.event.inputs.skip_build != 'true'
141-
run: |
142-
scp -i ~/.ssh/deploy_key \
143-
build-output/codecrow-images.tar.zst \
144-
${{ secrets.DEPLOY_USER }}@${{ secrets.DEPLOY_HOST }}:${{ env.DEPLOY_PATH }}/releases/
145-
146133
- name: Deploy on server
147134
run: |
148135
ssh -i ~/.ssh/deploy_key \
149136
${{ secrets.DEPLOY_USER }}@${{ secrets.DEPLOY_HOST }} \
150-
"chmod +x ${{ env.DEPLOY_PATH }}/server-deploy.sh && ${{ env.DEPLOY_PATH }}/server-deploy.sh"
137+
"chmod +x ${{ env.DEPLOY_PATH }}/server-deploy.sh && GITHUB_REPOSITORY_OWNER=${{ github.repository_owner }} ${{ env.DEPLOY_PATH }}/server-deploy.sh"
151138
152139
- name: Verify deployment
153140
run: |

‎deployment/build/production-build.sh‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,20 +11,20 @@ CONFIG_PATH="deployment/config"
1111

1212
cd "$(dirname "$0")/../../"
1313

14-
# echo "--- 1. Ensuring frontend submodule is synchronized ---"
15-
# if [ -d "$FRONTEND_DIR" ] && [ ! -f "$FRONTEND_DIR/.git" ]; then
16-
# echo "Stale frontend directory detected (not a submodule). Removing and re-initializing..."
17-
# rm -rf "$FRONTEND_DIR"
18-
# git submodule update --init -- "$FRONTEND_DIR"
19-
# elif [ ! -d "$FRONTEND_DIR" ]; then
20-
# echo "Initializing frontend submodule..."
21-
# git submodule update --init -- "$FRONTEND_DIR"
22-
# else
23-
# echo "Frontend submodule exists."
24-
# fi
25-
# echo "Fetching latest from origin and resetting to origin/$FRONTEND_BRANCH..."
26-
# (cd "$FRONTEND_DIR" && git fetch origin "$FRONTEND_BRANCH" && git reset --hard "origin/$FRONTEND_BRANCH")
27-
# echo "Frontend at: $(cd "$FRONTEND_DIR" && git log --oneline -1)"
14+
echo "--- 1. Ensuring frontend submodule is synchronized ---"
15+
if [ -d "$FRONTEND_DIR" ] && [ ! -f "$FRONTEND_DIR/.git" ]; then
16+
echo "Stale frontend directory detected (not a submodule). Removing and re-initializing..."
17+
rm -rf "$FRONTEND_DIR"
18+
git submodule update --init -- "$FRONTEND_DIR"
19+
elif [ ! -d "$FRONTEND_DIR" ]; then
20+
echo "Initializing frontend submodule..."
21+
git submodule update --init -- "$FRONTEND_DIR"
22+
else
23+
echo "Frontend submodule exists."
24+
fi
25+
echo "Fetching latest from origin and resetting to origin/$FRONTEND_BRANCH..."
26+
(cd "$FRONTEND_DIR" && git fetch origin "$FRONTEND_BRANCH" && git reset --hard "origin/$FRONTEND_BRANCH")
27+
echo "Frontend at: $(cd "$FRONTEND_DIR" && git log --oneline -1)"
2828

2929
echo "--- 2. Injecting Environment Configurations ---"
3030

‎deployment/ci/ci-build.sh‎

Lines changed: 17 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,7 @@
55
# 1. Writes .env files from GitHub secrets
66
# 2. Builds & tests Java artifacts (Maven) — fails fast on test errors
77
# 3. Copies MCP JARs to inference-orchestrator context
8-
# 4. Builds all 5 Docker images
9-
# 5. Saves them to a single compressed tarball
8+
# 4. Builds all 5 Docker images and pushes them to GHCR
109
#
1110
# Required env vars (set by GH Actions):
1211
# ENV_INFERENCE_ORCHESTRATOR — contents of inference-orchestrator/.env
@@ -21,10 +20,11 @@ ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
2120
MCP_JAR="java-ecosystem/mcp-servers/vcs-mcp/target/codecrow-vcs-mcp-1.0.jar"
2221
PLATFORM_MCP_JAR="java-ecosystem/mcp-servers/platform-mcp/target/codecrow-platform-mcp-1.0.jar"
2322
JAVA_DIR="java-ecosystem"
24-
OUTPUT_DIR="$ROOT_DIR/build-output"
23+
# GITHUB_REPOSITORY_OWNER is assumed to be provided for ghcr.io paths
24+
REPO_OWNER=${GITHUB_REPOSITORY_OWNER:-codecrow}
25+
REPO_OWNER=$(echo "$REPO_OWNER" | tr '[:upper:]' '[:lower:]')
2526

2627
cd "$ROOT_DIR"
27-
mkdir -p "$OUTPUT_DIR"
2828

2929
echo "=========================================="
3030
echo " CodeCrow CI Build"
@@ -83,41 +83,33 @@ for entry in "${IMAGES[@]}"; do
8383
IFS='|' read -r IMAGE_NAME CONTEXT DOCKERFILE <<< "$entry"
8484
# Scope cache per image to avoid collisions
8585
SCOPE="$(echo "$IMAGE_NAME" | tr '/' '-')"
86-
echo " Building $IMAGE_NAME from $CONTEXT ..."
86+
87+
# Map codecrow to ghcr.io/<repo-owner>/codecrow-<service>
88+
# E.g. codecrow/web-server -> ghcr.io/username/codecrow-web-server:latest
89+
SERVICE_NAME=$(echo "$IMAGE_NAME" | cut -d'/' -f2)
90+
FULL_IMAGE_NAME="ghcr.io/$REPO_OWNER/codecrow-$SERVICE_NAME:latest"
91+
92+
echo " Building and pushing $FULL_IMAGE_NAME from $CONTEXT ..."
8793
if [ -n "${DOCKERFILE:-}" ]; then
8894
docker buildx build \
8995
--cache-from "type=gha,scope=$SCOPE" \
9096
--cache-to "type=gha,mode=max,scope=$SCOPE" \
91-
--load \
92-
-t "${IMAGE_NAME}:latest" \
97+
--push \
98+
-t "$FULL_IMAGE_NAME" \
9399
-f "$CONTEXT/$DOCKERFILE" \
94100
"$CONTEXT"
95101
else
96102
docker buildx build \
97103
--cache-from "type=gha,scope=$SCOPE" \
98104
--cache-to "type=gha,mode=max,scope=$SCOPE" \
99-
--load \
100-
-t "${IMAGE_NAME}:latest" \
105+
--push \
106+
-t "$FULL_IMAGE_NAME" \
101107
"$CONTEXT"
102108
fi
103-
echo " ✓ $IMAGE_NAME built"
109+
echo " ✓ $FULL_IMAGE_NAME built and pushed"
104110
done
105111

106-
# ── 5. Save images to tarball ─────────────────────────────────────────────
107-
echo "--- 5. Saving Docker images to tarball ---"
108-
109-
IMAGE_LIST=""
110-
for entry in "${IMAGES[@]}"; do
111-
IFS='|' read -r IMAGE_NAME _ _ <<< "$entry"
112-
IMAGE_LIST="$IMAGE_LIST ${IMAGE_NAME}:latest"
113-
done
114-
115-
docker save $IMAGE_LIST | zstd -T0 --ultra -20 -o "$OUTPUT_DIR/codecrow-images.tar.zst"
116-
117-
TARBALL_SIZE=$(du -h "$OUTPUT_DIR/codecrow-images.tar.zst" | cut -f1)
118-
echo " ✓ Tarball created: codecrow-images.tar.zst ($TARBALL_SIZE)"
119-
120112
echo ""
121113
echo "=========================================="
122-
echo " Build complete! Artifact: build-output/codecrow-images.tar.zst"
114+
echo " Build and push complete!"
123115
echo "=========================================="

‎deployment/ci/server-deploy.sh‎

Lines changed: 14 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -2,47 +2,37 @@
22
###############################################################################
33
# server-deploy.sh — Runs ON THE LIVE SERVER via SSH from GitHub Actions.
44
#
5-
# Expects the tarball to already be uploaded to /opt/codecrow/releases/
5+
# Expects docker-compose.prod.yml to be updated to point to GHCR.
66
#
77
# Flow:
88
# 1. Pre-flight config checks
99
# 2. Backup PostgreSQL database
10-
# 3. Load new Docker images from tarball
10+
# 3. Pull new Docker images from Registry
1111
# 4. Stop existing services
1212
# 5. Start services (--no-build, --wait for healthchecks)
1313
# 6. Verify health
14-
# 7. Cleanup old releases & backups
14+
# 7. Cleanup old backups
1515
#
1616
# Usage:
17-
# server-deploy.sh [tarball-name]
18-
#
19-
# Default tarball: codecrow-images.tar.zst
17+
# GITHUB_REPOSITORY_OWNER=username server-deploy.sh
2018
###############################################################################
2119
set -euo pipefail
2220

2321
DEPLOY_DIR="/opt/codecrow"
24-
RELEASES_DIR="$DEPLOY_DIR/releases"
2522
CONFIG_DIR="$DEPLOY_DIR/config"
2623
BACKUP_DIR="$DEPLOY_DIR/backups"
2724
COMPOSE_FILE="$DEPLOY_DIR/docker-compose.prod.yml"
28-
TARBALL_NAME="${1:-codecrow-images.tar.zst}"
29-
TARBALL_PATH="$RELEASES_DIR/$TARBALL_NAME"
25+
26+
# For GHCR pulling
27+
export GITHUB_REPOSITORY_OWNER="${GITHUB_REPOSITORY_OWNER:-codecrow}"
28+
export GITHUB_REPOSITORY_OWNER=$(echo "$GITHUB_REPOSITORY_OWNER" | tr '[:upper:]' '[:lower:]')
3029

3130
echo "=========================================="
3231
echo " CodeCrow Server Deployment"
3332
echo " $(date '+%Y-%m-%d %H:%M:%S')"
3433
echo "=========================================="
3534

3635
# ── Pre-flight checks ─────────────────────────────────────────────────────
37-
if ! command -v zstd &>/dev/null; then
38-
echo "ERROR: zstd is not installed. Run: sudo apt-get install zstd"
39-
exit 1
40-
fi
41-
42-
if [ ! -f "$TARBALL_PATH" ]; then
43-
echo "ERROR: Tarball not found: $TARBALL_PATH"
44-
exit 1
45-
fi
4636

4737
if [ ! -f "$COMPOSE_FILE" ]; then
4838
echo "ERROR: docker-compose.prod.yml not found: $COMPOSE_FILE"
@@ -94,10 +84,10 @@ else
9484
echo " ⚠ PostgreSQL not running — skipping backup (first deploy?)"
9585
fi
9686

97-
# ── 2. Load Docker images ─────────────────────────────────────────────────
98-
echo "--- 2. Loading Docker images from tarball ---"
99-
zstd -d --stdout "$TARBALL_PATH" | docker load
100-
echo " ✓ Images loaded"
87+
# ── 2. Pull Docker images ─────────────────────────────────────────────────
88+
echo "--- 2. Pulling Docker images from registry ---"
89+
docker compose -f "$COMPOSE_FILE" pull
90+
echo " ✓ Images pulled"
10191

10292
# ── 3. Stop existing services ─────────────────────────────────────────────
10393
echo "--- 3. Stopping existing services ---"
@@ -129,12 +119,8 @@ echo " ✓ Services started and healthy"
129119
echo "--- 5. Service status ---"
130120
docker compose -f docker-compose.prod.yml ps
131121

132-
# ── 6. Cleanup old releases (keep last 5) ─────────────────────────────────
133-
echo "--- 6. Cleaning up old releases and backups ---"
134-
cd "$RELEASES_DIR"
135-
ls -1t *.tar.zst 2>/dev/null | tail -n +6 | xargs -r rm -f
136-
REMAINING=$(ls -1 *.tar.zst 2>/dev/null | wc -l)
137-
echo " ✓ Keeping $REMAINING release(s)"
122+
# ── 6. Cleanup old backups ────────────────────────────────────────────────
123+
echo "--- 6. Cleaning up old backups ---"
138124

139125
# Cleanup old DB backups (keep last 10)
140126
cd "$BACKUP_DIR"

‎deployment/docker-compose.prod.yml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ services:
6060
restart: unless-stopped
6161

6262
web-server:
63-
image: codecrow/web-server:latest
63+
image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-rostilos}/codecrow-web-server:latest
6464
container_name: codecrow-web-application
6565
environment:
6666
SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/${POSTGRES_DB:-codecrow_ai}
@@ -113,7 +113,7 @@ services:
113113
restart: unless-stopped
114114

115115
pipeline-agent:
116-
image: codecrow/pipeline-agent:latest
116+
image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-rostilos}/codecrow-pipeline-agent:latest
117117
container_name: codecrow-pipeline-agent
118118
environment:
119119
SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/${POSTGRES_DB:-codecrow_ai}
@@ -165,7 +165,7 @@ services:
165165
- "host.docker.internal:host-gateway"
166166

167167
inference-orchestrator:
168-
image: codecrow/inference-orchestrator:latest
168+
image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-rostilos}/codecrow-inference-orchestrator:latest
169169
container_name: codecrow-inference-orchestrator
170170
ports:
171171
- "127.0.0.1:8000:8000"
@@ -198,7 +198,7 @@ services:
198198
memory: 512M
199199

200200
rag-pipeline:
201-
image: codecrow/rag-pipeline:latest
201+
image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-rostilos}/codecrow-rag-pipeline:latest
202202
container_name: codecrow-rag-pipeline
203203
environment:
204204
CODECROW_WEB_SERVER_URL: http://web-server:8081
@@ -231,7 +231,7 @@ services:
231231
- "host.docker.internal:host-gateway"
232232

233233
web-frontend:
234-
image: codecrow/web-frontend:latest
234+
image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-rostilos}/codecrow-web-frontend:latest
235235
container_name: codecrow-web-frontend
236236
ports:
237237
- "127.0.0.1:8080:8080"

‎frontend‎

‎java-ecosystem/libs/analysis-engine/pom.xml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,18 @@
2727
<artifactId>codecrow-core</artifactId>
2828
</dependency>
2929

30+
<!-- Commit graph (DAG) module -->
31+
<dependency>
32+
<groupId>org.rostilos.codecrow</groupId>
33+
<artifactId>codecrow-commit-graph</artifactId>
34+
</dependency>
35+
36+
<!-- File content/snapshot module -->
37+
<dependency>
38+
<groupId>org.rostilos.codecrow</groupId>
39+
<artifactId>codecrow-file-content</artifactId>
40+
</dependency>
41+
3042
<!-- Analysis API interfaces -->
3143
<dependency>
3244
<groupId>org.rostilos.codecrow</groupId>
@@ -130,6 +142,8 @@
130142
@{argLine}
131143
--add-opens org.rostilos.codecrow.analysisengine/org.rostilos.codecrow.analysisengine.service=com.fasterxml.jackson.databind
132144
--add-opens org.rostilos.codecrow.core/org.rostilos.codecrow.core.model.branch=org.rostilos.codecrow.analysisengine
145+
--add-opens org.rostilos.codecrow.core/org.rostilos.codecrow.core.model.project=org.rostilos.codecrow.analysisengine
146+
--add-opens org.rostilos.codecrow.core/org.rostilos.codecrow.core.model.pullrequest=org.rostilos.codecrow.analysisengine
133147
</argLine>
134148
</configuration>
135149
</plugin>

‎java-ecosystem/libs/analysis-engine/src/main/java/module-info.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
requires spring.data.jpa;
99
requires spring.tx;
1010
requires org.rostilos.codecrow.core;
11+
requires org.rostilos.codecrow.commitgraph;
12+
requires org.rostilos.codecrow.filecontent;
1113
requires org.rostilos.codecrow.vcs;
1214
requires org.rostilos.codecrow.analysisapi;
1315
requires transitive org.rostilos.codecrow.queue;
@@ -32,7 +34,6 @@
3234
exports org.rostilos.codecrow.analysisengine.processor;
3335
exports org.rostilos.codecrow.analysisengine.processor.analysis;
3436
exports org.rostilos.codecrow.analysisengine.service;
35-
exports org.rostilos.codecrow.analysisengine.service.gitgraph;
3637
exports org.rostilos.codecrow.analysisengine.service.rag;
3738
exports org.rostilos.codecrow.analysisengine.service.vcs;
3839
exports org.rostilos.codecrow.analysisengine.util;
@@ -47,8 +48,6 @@
4748
to spring.core, spring.beans, spring.context, com.fasterxml.jackson.databind;
4849
opens org.rostilos.codecrow.analysisengine.service
4950
to spring.core, spring.beans, spring.context, com.fasterxml.jackson.databind;
50-
opens org.rostilos.codecrow.analysisengine.service.gitgraph
51-
to spring.core, spring.beans, spring.context, com.fasterxml.jackson.databind;
5251
opens org.rostilos.codecrow.analysisengine.service.rag
5352
to spring.core, spring.beans, spring.context, com.fasterxml.jackson.databind;
5453
opens org.rostilos.codecrow.analysisengine.service.vcs

0 commit comments

Comments
 (0)