Merge pull request #26 from UXcaptain/refactor-aws-s3-to-minio

REPLACE S3 storage from cloud AWS to self-hosted minIO

Author: SergioNR

.env.example

@@ -21,6 +21,11 @@ PORT=3000
 
 AWS_SECRET_ACCESS_KEY=
 AWS_ACCESS_KEY_ID=
-AWS_REGION=
-AWS_BUCKET=
-TRANSCRIPTION_ENABLED=
+S3_REGION=
+S3_BUCKET=
+S3_ENDPOINT=
+
+TRANSCRIPTION_ENABLED=
+
+MINIO_ROOT_USER=
+MINIO_ROOT_PASSWORD=

.kilocode/rules/memory-bank/tech.md

@@ -89,8 +89,8 @@ PRISMA_POSTGRES_CONNECTION_STRING=postgresql://user:password@localhost:5432/uxca
 # AWS S3
 AWS_ACCESS_KEY_ID=your-access-key
 AWS_SECRET_ACCESS_KEY=your-secret-key
-AWS_REGION=eu-west-3
-AWS_BUCKET=your-bucket-name
+S3_REGION=eu-west-3
+S3_BUCKET=your-bucket-name
 
 # Stripe
 STRIPE_API_KEY=sk_test_...

compose.yaml

@@ -1,38 +1,56 @@
 name: uxcaptain
 services:
-  server:
-    container_name: server
-    build:
-      context: .
-      dockerfile: Dockerfile.dev
-    environment:
-      NODE_ENV: ${NODE_ENV}
-      DEPLOY_ENVIRONMENT: ${DEPLOY_ENVIRONMENT}
-      POSTHOG_API_KEY: ${POSTHOG_API_KEY}
-      SESSION_SECRET: ${SESSION_SECRET}
-      BREVO_API_KEY: ${BREVO_API_KEY}
-      PRISMA_POSTGRES_DIRECT_URL: ${PRISMA_POSTGRES_DIRECT_URL}
-      PRISMA_POSTGRES_CONNECTION_STRING: ${PRISMA_POSTGRES_CONNECTION_STRING}
-      TELEGRAM_BOT_API_KEY: ${TELEGRAM_BOT_API_KEY}
-      FRONT_WEB_APP_ORIGIN_URL: ${FRONT_WEB_APP_ORIGIN_URL}
-      STRIPE_API_KEY: ${STRIPE_API_KEY}
-      STRIPE_WEBHOOK_SECRET: ${STRIPE_WEBHOOK_SECRET}
-      PORT: ${PORT}
-      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
-      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
-      AWS_BUCKET: dev-analysis-entry-storage
-      AWS_REGION: ${AWS_REGION}
+  # server:
+  #   container_name: server
+  #   build:
+  #     context: .
+  #     dockerfile: Dockerfile.dev
+  #   environment:
+  #     NODE_ENV: ${NODE_ENV}
+  #     DEPLOY_ENVIRONMENT: ${DEPLOY_ENVIRONMENT}
+  #     POSTHOG_API_KEY: ${POSTHOG_API_KEY}
+  #     SESSION_SECRET: ${SESSION_SECRET}
+  #     BREVO_API_KEY: ${BREVO_API_KEY}
+  #     PRISMA_POSTGRES_DIRECT_URL: ${PRISMA_POSTGRES_DIRECT_URL}
+  #     PRISMA_POSTGRES_CONNECTION_STRING: ${PRISMA_POSTGRES_CONNECTION_STRING}
+  #     TELEGRAM_BOT_API_KEY: ${TELEGRAM_BOT_API_KEY}
+  #     FRONT_WEB_APP_ORIGIN_URL: ${FRONT_WEB_APP_ORIGIN_URL}
+  #     STRIPE_API_KEY: ${STRIPE_API_KEY}
+  #     STRIPE_WEBHOOK_SECRET: ${STRIPE_WEBHOOK_SECRET}
+  #     PORT: ${PORT}
+  #     AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+  #     AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+  #     S3_BUCKET: dev-analysis-entry-storage
+  #     S3_REGION: ${S3_REGION}
 
+  #   ports:
+  #     - 3000:3000
+  #   develop:
+  #     watch:
+  #       - path: .
+  #         action: sync+restart #* Synchronize code changes and restart the server
+  #         target: /usr/src/app/
+  #   depends_on: #* References the SERVICE name - NOT the container_name
+  #     database:
+  #       condition: service_started
+  #     minio:
+          # condition: service_started
+  #   networks:
+  #     - uxcaptain-network
+  
+  minio:
+    image: minio/minio:latest
+    container_name: minio
+    restart: on-failure
+    environment:
+      - MINIO_ROOT_USER=${MINIO_ROOT_USER}
+      - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD}
+    volumes:
+      - /Users/martaperezsanchez/repos/minio:/data
     ports:
-      - 3000:3000
-    develop:
-      watch:
-        - path: .
-          action: sync+restart #* Synchronize code changes and restart the server
-          target: /usr/src/app/
-    depends_on: #* References the SERVICE name - NOT the container_name
-      database:
-        condition: service_started
+      - 9000:9000
+      - 9001:9001
+    command: server /data --console-address ":9001"
     networks:
       - uxcaptain-network
 

server/controllers/analysisEntryController.js

@@ -1,4 +1,4 @@
-import { generateS3GetPresignedUrl, generateS3PutPresignedUrl } from '../integrations/aws/s3.js';
+import { generateS3GetPresignedUrl } from '../integrations/aws/s3.js';
 import { createAnalysisEntryInDb, getAnalysisEntryDetailsById, markAnalysisEntryAsSubmitted } from '../models/analysisEntryModel.js';
 import { processTranscriptionRequest } from '../services/analysisService.js';
 
@@ -71,17 +71,3 @@ export const getAnalysisEntryDetails = async (req, res) => {
     transcriptionSegments: analysisEntryDetails.transcription_segments,
   });
 };
-
-export const getAnalysisEntryPresignedUploadUrl = async (req, res) => {
-  const { analysisEntryId, analysisId } = req.body;
-
-  const key = `analysis/${analysisId}/${analysisEntryId}/recording.mp4`;
-
-  const analysisEntryPresignedUrl = await generateS3PutPresignedUrl(key);
-
-  return res.status(200).json({
-    success: true,
-    message: 'PresignedUploadUrl retrieved successfully',
-    analysisEntryPresignedUploadUrl: analysisEntryPresignedUrl,
-  });
-};

server/integrations/aws/Transcribe.js

@@ -7,16 +7,16 @@ import {
 import { getS3Object } from './s3.js';
 import { logInfo } from '../../config/loggerFunctions.js';
 
-const transcribeClient = new TranscribeClient({ region: process.env.AWS_REGION });
+const transcribeClient = new TranscribeClient({ region: process.env.S3_REGION });
 
 export const requestAnalysisEntryTranscriptionToAWSTranscribe = async (transcriptionRequest) => {
   const command = new StartTranscriptionJobCommand({
     TranscriptionJobName: transcriptionRequest.analysisEntryId,
     LanguageCode: transcriptionRequest.languageCode,
     Media: {
-      MediaFileUri: `s3://${process.env.AWS_BUCKET}/analysis/${transcriptionRequest.analysisId}/${transcriptionRequest.analysisEntryId}/recording.mp4`,
+      MediaFileUri: `s3://${process.env.S3_BUCKET}/analysis/${transcriptionRequest.analysisId}/${transcriptionRequest.analysisEntryId}/recording.mp4`,
     },
-    OutputBucketName: process.env.AWS_BUCKET,
+    OutputBucketName: process.env.S3_BUCKET,
     OutputKey: `analysis/${transcriptionRequest.analysisId}/${transcriptionRequest.analysisEntryId}/transcription.json`,
   });
 

server/integrations/aws/s3.js

@@ -2,16 +2,18 @@ import { S3Client, GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 
 export const s3client = new S3Client({
-  region: process.env.AWS_REGION,
-  credentials: { // https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/modules/credentials.html
-    accessKeyId: process.env.AWS_ACCESS_KEY_ID,
-    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+  region: process.env.S3_REGION, // irrelevant since  miniIO doesnt takei into account
+  endpoint: process.env.S3_ENDPOINT, // Container network endpoint
+  forcePathStyle: true, // Required for MinIO path-style URLs
+  credentials: {
+    accessKeyId: process.env.MINIO_ROOT_USER,
+    secretAccessKey: process.env.MINIO_ROOT_PASSWORD,
   },
 });
 
 export const generateS3GetPresignedUrl = async (key) => {
   const command = new GetObjectCommand({
-    Bucket: process.env.AWS_BUCKET,
+    Bucket: process.env.S3_BUCKET,
     Key: key,
   });
 
@@ -22,7 +24,7 @@ export const generateS3GetPresignedUrl = async (key) => {
 
 export const generateS3PutPresignedUrl = async (key) => {
   const command = new PutObjectCommand({
-    Bucket: process.env.AWS_BUCKET,
+    Bucket: process.env.S3_BUCKET,
     Key: key,
     ContentType: 'video/mp4',
   });
@@ -36,7 +38,7 @@ export const generateS3PutPresignedUrl = async (key) => {
 
 export const getS3Object = async (key) => {
   const command = new GetObjectCommand({
-    Bucket: process.env.AWS_BUCKET,
+    Bucket: process.env.S3_BUCKET,
     Key: key,
   });