Modular seperation for server and consumer

.github/workflows/push-images.yml

@@ -1,7 +1,9 @@
 name: Build and push images to GHCR
 
 on:
-  workflow_dispatch:
+  push:
+    branches:
+      - main
 
 permissions:
   contents: read

.github/workflows/server-tests.yml

@@ -0,0 +1,66 @@
+name: Unit Tests for Server
+
+on:
+  pull_request:
+    branches:
+      - staging
+    paths:
+      - 'server/**'
+  push:
+    branches:
+      - staging
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Setup Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '20'
+        cache: 'npm'
+
+    - name: Install server dependencies
+      run: npm install --ignore-scripts
+      working-directory: server
+
+    - name: Run Unit Tests & Generate Coverage
+      run: npm run test:unit -- --coverage
+      working-directory: server
+      env:
+        NODE_ENV: test
+        REDIS_PWD: ${{ secrets.REDIS_PWD }}
+        REDIS_HOST: ${{ secrets.REDIS_HOST }}
+        REDIS_PORT: ${{ secrets.REDIS_PORT }}
+        SUPABASE_DB_URL: ${{ secrets.SUPABASE_DB_URL }}
+
+    - name: Upload coverage reports
+      uses: codecov/codecov-action@ab904c41d6ece82784817410c45d8b8c02684457
+      with:
+        directory: server/coverage
+        fail_ci_if_error: false
+
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Setup Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '20'
+        cache: 'npm'
+
+    - name: Install dependencies
+      run: npm install --ignore-scripts
+      working-directory: server
+
+    - name: Run ESLint
+      run: npx eslint server --ext .js
+      continue-on-error: true

.github/workflows/sonarqube.yml

@@ -1,7 +1,10 @@
 name: SonarQube Scan
 
 on:
-  workflow_dispatch:
+  workflow_run:
+    workflows: ["Unit Tests for Server"]
+    types:
+      - completed
 
 jobs:
   scan:
@@ -18,14 +21,20 @@ jobs:
           node-version: 18
 
       - name: Install dependencies
-        run: npm install
+        run: npm install --ignore-scripts
 
       # Optional: run tests to generate coverage file
       #- name: Run tests
       #  run: npm test -- --coverage
 
       - name: SonarQube Scan
         uses: sonarsource/sonarqube-scan-action@v2
+        with:
+          projectBaseDir: server
+          args: >
+            -Dsonar.organization=${{ secrets.SONAR_ORGANIZATION }}
+            -Dsonar.projectKey=${{ secrets.SONAR_PROJECT_KEY }}
+            -Dsonar.javascript.lcov.reportPaths=coverage/lcov.info
         env:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
           SONAR_HOST_URL: https://sonarcloud.io

README.md

@@ -49,22 +49,10 @@ Stop all instances
 pm2 stop all
 ```
 
-### Testing?
+### Testing >>
 
+Server Unit Tests
 ```bash
-
-```
-
-
-
-
-https://www.youtube.com/watch?v=jgpVdJB2sKQ
-
-aks tutorial 
-https://www.youtube.com/watch?v=dkPsslnKK1Y
-
-network chuck kubernetes
-https://www.youtube.com/watch?v=7bA0gTroJjw
-
-create aks cluter in azure
-https://www.youtube.com/watch?v=RUoejLILgyA
+cd server
+npm run test:unit
+```

consumer/Dockerfile.consumer

@@ -7,8 +7,13 @@ WORKDIR /app
 COPY package*.json ./
 RUN npm ci --only=production
 
-# Copy app code
-COPY consumer/consumer.js ./
+# Copy consumer modular codebase structure
+COPY consumer/ ./consumer/
+
+# Copy config directory (for .env file structure)
+# Note: .env file should be provided via environment variables in production
+# The config directory should exist in the repo (even if empty)
+COPY config/ ./config/
 
 # Small health-check-friendly default command
-CMD ["node", "consumer.js"]
+CMD ["node", "consumer/consumer.js"]

consumer/config/index.js

@@ -0,0 +1,89 @@
+/**
+ * Configuration management for the consumer application
+ * Centralizes all environment variables and provides validation
+ */
+
+import dotenv from 'dotenv';
+import crypto from 'node:crypto';
+
+// Load environment variables
+dotenv.config({ path: '../config/.env' });
+
+/**
+ * Redis configuration
+ */
+export const redisConfig = {
+  url: process.env.REDIS_URL || `redis://:${process.env.REDIS_PWD}@${process.env.REDIS_HOST}:${process.env.REDIS_PORT}`,
+  stream: process.env.STREAM_KEY || "chat_stream",
+  group: process.env.CONSUMER_GROUP || "cg1",
+consumer: process.env.CONSUMER_NAME || `c-${Array.from(crypto.getRandomValues(new Uint8Array(6)), byte => (byte % 36).toString(36)).join('')}`,
+};
+
+/**
+ * Postgres configuration
+ */
+export const postgresConfig = {
+  connectionString: process.env.SUPABASE_DB_URL,
+  ssl: { rejectUnauthorized: false },
+  max: 5,
+  idleTimeoutMillis: 30000,
+  connectionTimeoutMillis: 20000,
+};
+
+/**
+ * Consumer settings
+ */
+export const consumerSettings = {
+  // Redis stream settings
+  readCount: Number(process.env.XREAD_COUNT || 500),
+  readBlockMs: Number(process.env.XREAD_BLOCK_MS || 10),
+
+  // Batch processing settings
+  batchSize: Number(process.env.BATCH_SIZE || 200),
+  batchTimeoutMs: Number(process.env.BATCH_TIMEOUT_MS || 200),
+  flushConcurrency: Number(process.env.FLUSH_CONCURRENCY || 1),
+
+  // Pending message recovery settings
+  claimMinIdleMs: Number(process.env.CLAIM_MIN_IDLE_MS || 10000),
+  claimCount: Number(process.env.CLAIM_COUNT || 200),
+};
+
+/**
+ * Logging configuration
+ */
+export const LOG_CONFIG = {
+  LEVEL: process.env.LOG_LEVEL || 'info',
+  COLORS: true
+};
+
+/**
+ * Validate required configuration
+ */
+export function validateConfig() {
+  const required = {
+    REDIS_URL: redisConfig.url,
+    SUPABASE_DB_URL: postgresConfig.connectionString,
+  };
+
+  const missing = Object.entries(required)
+    .filter(([key, value]) => !value)
+    .map(([key]) => key);
+
+  if (missing.length > 0) {
+    throw new Error(`Missing required environment variables: ${missing.join(', ')}`);
+  }
+}
+
+/**
+ * Get all configuration
+ */
+export function getConfig() {
+  return {
+    redis: redisConfig,
+    postgres: postgresConfig,
+    settings: consumerSettings,
+  };
+}
+
+// Validate configuration on import
+validateConfig();