diff --git a/.dockerignore b/.dockerignore index 10321c48..9514a1bf 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,8 +1,15 @@ .git .github .gradle +.env +secrets build out +logs +load-testing/results/ +*.jfr +*.hprof +gc.log* *.iml .idea **/.DS_Store diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 620ddb46..d5a6a932 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -1,4 +1,7 @@ # CI/CD에 필요한 GitHub Secrets (Repo Settings > Secrets and variables > Actions) +# - Docker Hub: DOCKERHUB_USERNAME, DOCKERHUB_TOKEN +# - Server: SERVER_HOST, SERVER_USER, SERVER_SSH_KEY, SERVER_SSH_PORT +# - GitHub repo/submodule access: GITHUB_TOKEN name: cicd @@ -40,8 +43,8 @@ jobs: - name: Run tests run: | - echo ">>> [CI] Running tests..." - ./gradlew test --no-daemon + echo ">>> [CI] Running tests without integration tests..." + ./gradlew test -PskipIntegrationTests --no-daemon echo ">>> [CI] Tests passed" docker-build-and-push: @@ -118,18 +121,6 @@ jobs: IMAGE: ${{ secrets.DOCKERHUB_USERNAME }}/dorumdorum-be:pinpoint DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - FIREBASE_SERVICE_ACCOUNT_B64: ${{ secrets.FIREBASE_SERVICE_ACCOUNT_B64 }} - RDB_USERNAME: ${{ secrets.RDB_USERNAME }} - RDB_URL: ${{ secrets.RDB_URL }} - RDB_PASSWORD: ${{ secrets.RDB_PASSWORD }} - MYSQL_ROOT_PASSWORD: ${{ secrets.MYSQL_ROOT_PASSWORD }} - MYSQL_DATABASE: ${{ secrets.MYSQL_DATABASE }} - JWT_KEY: ${{ secrets.JWT_KEY }} - JWT_ACCESS_EXPIRATION: ${{ secrets.JWT_ACCESS_EXPIRATION }} - JWT_REFRESH_EXPIRATION: ${{ secrets.JWT_REFRESH_EXPIRATION }} - SMTP_USERNAME: ${{ secrets.SMTP_USERNAME }} - SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }} - DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} CONTAINER_NAME: dorumdorum-be GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO_FULL_NAME: ${{ github.repository }} @@ -138,56 +129,11 @@ jobs: username: ${{ secrets.SERVER_USER }} key: ${{ secrets.SERVER_SSH_KEY }} port: ${{ secrets.SERVER_SSH_PORT }} - envs: IMAGE,DOCKERHUB_USERNAME,DOCKERHUB_TOKEN,FIREBASE_SERVICE_ACCOUNT_B64,RDB_USERNAME,RDB_URL,RDB_PASSWORD,MYSQL_ROOT_PASSWORD,MYSQL_DATABASE,JWT_KEY,JWT_ACCESS_EXPIRATION,JWT_REFRESH_EXPIRATION,SMTP_USERNAME,SMTP_PASSWORD,CONTAINER_NAME,GITHUB_TOKEN,REPO_FULL_NAME,DISCORD_WEBHOOK + envs: IMAGE,DOCKERHUB_USERNAME,DOCKERHUB_TOKEN,CONTAINER_NAME,GITHUB_TOKEN,REPO_FULL_NAME script_stop: true script: | set -e - echo ">>> [Deploy] SSH 연결됨, 환경 변수 정규화 중..." - - RDB_USERNAME="$(printf '%s' "$RDB_USERNAME" | tr -d '\r\n')" - RDB_URL="$(printf '%s' "$RDB_URL" | tr -d '\r\n')" - RDB_PASSWORD="$(printf '%s' "$RDB_PASSWORD" | tr -d '\r\n')" - MYSQL_ROOT_PASSWORD="$(printf '%s' "$MYSQL_ROOT_PASSWORD" | tr -d '\r\n')" - MYSQL_DATABASE="$(printf '%s' "$MYSQL_DATABASE" | tr -d '\r\n')" - JWT_KEY="$(printf '%s' "$JWT_KEY" | tr -d '\r\n')" - JWT_ACCESS_EXPIRATION="$(printf '%s' "$JWT_ACCESS_EXPIRATION" | tr -d '\r\n')" - JWT_REFRESH_EXPIRATION="$(printf '%s' "$JWT_REFRESH_EXPIRATION" | tr -d '\r\n')" - SMTP_USERNAME="$(printf '%s' "$SMTP_USERNAME" | tr -d '\r\n')" - SMTP_PASSWORD="$(printf '%s' "$SMTP_PASSWORD" | tr -d '\r\n')" - - DEPLOY_PATH="${HOME}/dorumdorum" - mkdir -p "$DEPLOY_PATH" - cd "$DEPLOY_PATH" - echo ">>> [Deploy] DEPLOY_PATH=$DEPLOY_PATH" - - # 1) Firebase 서비스 계정 파일 생성 (VM에 저장) - printf '%s' "$FIREBASE_SERVICE_ACCOUNT_B64" | base64 -d > firebase-service-account.json - chmod 600 firebase-service-account.json - - # 2) 컨테이너에서 참조할 경로를 env로 고정 - FIREBASE_SERVICE_ACCOUNT_PATH="/app/firebase-service-account.json" - - # 3) .env 생성 - # Pinpoint Web reads Spring datasource keys directly. - printf '%s\n' \ - "SPRING_PROFILES_ACTIVE=prod" \ - "RDB_USERNAME=$RDB_USERNAME" \ - "RDB_URL=$RDB_URL" \ - "RDB_PASSWORD=$RDB_PASSWORD" \ - "MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASSWORD" \ - "MYSQL_DATABASE=$MYSQL_DATABASE" \ - "SPRING_DATASOURCE_URL=$RDB_URL" \ - "SPRING_DATASOURCE_USERNAME=$RDB_USERNAME" \ - "SPRING_DATASOURCE_PASSWORD=$RDB_PASSWORD" \ - "JWT_KEY=$JWT_KEY" \ - "JWT_ACCESS_EXPIRATION=$JWT_ACCESS_EXPIRATION" \ - "JWT_REFRESH_EXPIRATION=$JWT_REFRESH_EXPIRATION" \ - "SMTP_USERNAME=$SMTP_USERNAME" \ - "SMTP_PASSWORD=$SMTP_PASSWORD" \ - "DISCORD_WEBHOOK=$DISCORD_WEBHOOK" \ - "FIREBASE_SERVICE_ACCOUNT_PATH=$FIREBASE_SERVICE_ACCOUNT_PATH" > .env - chmod 600 .env - echo ">>> [Deploy] .env, firebase-service-account.json 생성 완료" + echo ">>> [Deploy] SSH 연결됨" docker info >/dev/null 2>&1 || { echo "Docker socket permission denied for current deploy user." @@ -198,8 +144,8 @@ jobs: echo "$DOCKERHUB_TOKEN" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin echo ">>> [Deploy] Docker Hub 로그인 완료" - echo ">>> [Deploy] MySQL 이미지 pull 중..." - docker pull mysql:8.0 + echo ">>> [Deploy] PostgreSQL 이미지 pull 중..." + docker pull postgres:16-alpine echo ">>> [Deploy] Redis 이미지 pull 중..." docker pull redis:7-alpine echo ">>> [Deploy] Backend 이미지 pull 중..." @@ -215,9 +161,14 @@ jobs: echo ">>> [Deploy] COMPOSE_PATH=$COMPOSE_PATH, Git clone 중..." rm -rf temp_repo - # Git에서 production 브랜치로 docker-compose.yml, monitoring 가져오기 + # Git에서 production 브랜치로 docker-compose.yml, monitoring, secrets submodule 가져오기 git clone --depth 1 -b production "https://x-access-token:${GITHUB_TOKEN}@github.com/${REPO_FULL_NAME}.git" temp_repo - echo ">>> [Deploy] Clone 완료, docker-compose.yml · monitoring 복사 중..." + ( + cd temp_repo + git -c url."https://x-access-token:${GITHUB_TOKEN}@github.com/".insteadOf="https://github.com/" \ + submodule update --init --recursive --depth 1 + ) + echo ">>> [Deploy] Clone 완료, docker-compose.yml · monitoring · secrets 복사 중..." if [ -f "temp_repo/BE/docker-compose.yml" ]; then cp temp_repo/BE/docker-compose.yml . @@ -248,6 +199,19 @@ jobs: mkdir -p monitoring/prometheus monitoring/grafana/provisioning/dashboards/json monitoring/grafana/provisioning/datasources fi + rm -rf secrets + if [ -d "temp_repo/BE/secrets" ]; then + cp -R temp_repo/BE/secrets . + echo ">>> [Deploy] BE/secrets 복사 완료" + elif [ -d "temp_repo/secrets" ]; then + cp -R temp_repo/secrets . + echo ">>> [Deploy] secrets 복사 완료" + else + echo "ERROR: secrets submodule directory not found in temp_repo/" + exit 1 + fi + chmod 600 secrets/.env secrets/firebase-service-account.json + rm -rf temp_repo echo ">>> [Deploy] temp_repo 삭제, compose 파일 확인 중..." @@ -257,21 +221,23 @@ jobs: ls -la exit 1 fi - - # .env 파일과 firebase-service-account.json 복사 - cp "$DEPLOY_PATH/.env" . - cp "$DEPLOY_PATH/firebase-service-account.json" . - echo ">>> [Deploy] .env, firebase-service-account.json 복사 완료" + + if [ ! -f "secrets/.env" ] || [ ! -f "secrets/firebase-service-account.json" ]; then + echo "ERROR: required secret files are missing" + ls -la secrets || true + exit 1 + fi # docker-compose로 모든 서비스 실행 export BACKEND_IMAGE="$IMAGE" export CONTAINER_NAME="$CONTAINER_NAME" + COMPOSE="docker compose --env-file secrets/.env -f docker-compose.yml" echo ">>> [Deploy] 기존 컨테이너 정리 (down + 이름으로 강제 제거)..." - docker compose -f docker-compose.yml down --remove-orphans >/dev/null 2>&1 || true + $COMPOSE down --remove-orphans >/dev/null 2>&1 || true docker rm -f dorumdorum-redis dorumdorum-be dorumdorum-prometheus dorumdorum-grafana zoo1 pinpoint-hbase pinpoint-collector pinpoint-web 2>/dev/null || true echo ">>> [Deploy] [1/6] ZooKeeper 기동..." - docker compose -f docker-compose.yml up -d zoo1 + $COMPOSE up -d zoo1 for i in $(seq 1 30); do if docker ps --format '{{.Names}}' | grep -q '^zoo1$'; then echo ">>> [Deploy] ZooKeeper up" @@ -281,7 +247,7 @@ jobs: done echo ">>> [Deploy] [2/6] HBase 기동..." - docker compose -f docker-compose.yml up -d pinpoint-hbase + $COMPOSE up -d pinpoint-hbase echo ">>> [Deploy] HBase 초기화 대기..." HBASE_READY=0 for i in $(seq 1 300); do @@ -305,7 +271,7 @@ jobs: fi echo ">>> [Deploy] [3/6] Pinpoint Collector 기동..." - docker compose -f docker-compose.yml up -d pinpoint-collector + $COMPOSE up -d pinpoint-collector COLLECTOR_READY=0 for i in $(seq 1 60); do if docker logs pinpoint-collector 2>&1 | grep -Eq "Started .*CollectorApp|Started CollectorApp"; then @@ -322,7 +288,7 @@ jobs: fi echo ">>> [Deploy] [4/6] Pinpoint Web 기동..." - docker compose -f docker-compose.yml up -d pinpoint-web + $COMPOSE up -d pinpoint-web WEB_READY=0 for i in $(seq 1 60); do if docker logs pinpoint-web 2>&1 | grep -Eq "Started .*WebApp|Started WebApp|Started .*PinpointWebApplication"; then @@ -339,10 +305,10 @@ jobs: fi echo ">>> [Deploy] [5/6] Backend 기동..." - docker compose -f docker-compose.yml up -d backend + $COMPOSE up -d backend echo ">>> [Deploy] [6/6] Redis/Monitoring 기동..." - docker compose -f docker-compose.yml up -d redis prometheus grafana + $COMPOSE up -d redis prometheus grafana echo ">>> [Deploy] 오래된 이미지 정리 중..." docker image prune -af --filter "until=168h" diff --git a/.gitignore b/.gitignore index a311993f..7534d640 100644 --- a/.gitignore +++ b/.gitignore @@ -55,7 +55,16 @@ pinpoint-agent/tools/ .env /analysis/ +### JVM profiling artifacts ### +logs/ +*.jfr +*.hprof +gc.log* + +### k6 load test artifacts ### +load-testing/results/ + # macOS artefacts .DS_Store -docs/ \ No newline at end of file +docs/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..392a0f4e --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "secrets"] + path = secrets + url = https://github.com/DorumDorum/secrets diff --git a/Dockerfile b/Dockerfile index 83d51c1e..6daa913c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,8 @@ WORKDIR /app COPY --from=builder /app/build/libs/*.jar app.jar +RUN mkdir -p /app/logs/jfr + EXPOSE 8080 -ENTRYPOINT ["sh", "-c", "exec java ${JAVA_OPTS} -Dspring.profiles.active=${SPRING_PROFILES_ACTIVE:-prod} -jar /app/app.jar"] +ENTRYPOINT ["sh", "-c", "DEFAULT_JAVA_OPTS=\"-XX:StartFlightRecording=name=dorumdorum,settings=${JFR_SETTINGS:-profile},disk=true,dumponexit=true,filename=/app/logs/jfr/dorumdorum-${JFR_RUN_ID:-cloud-load-test}.jfr,maxage=${JFR_MAX_AGE:-30m},maxsize=${JFR_MAX_SIZE:-512m}\"; EFFECTIVE_JAVA_OPTS=\"${DEFAULT_JAVA_OPTS} ${JAVA_OPTS:-}\"; exec java ${EFFECTIVE_JAVA_OPTS} -Dspring.profiles.active=${SPRING_PROFILES_ACTIVE:-prod} -jar /app/app.jar"] diff --git a/Dockerfile.pinpoint b/Dockerfile.pinpoint index ccbffc8e..ca7d9574 100644 --- a/Dockerfile.pinpoint +++ b/Dockerfile.pinpoint @@ -28,6 +28,8 @@ WORKDIR /app COPY --from=builder /app/build/libs/*.jar app.jar COPY --from=builder /app/pinpoint-agent /app/pinpoint-agent +RUN mkdir -p /app/logs/jfr + # Pinpoint Agent JVM 옵션 (환경변수로 Collector IP 등 오버라이드 가능) ENV PINPOINT_VERSION=2.5.4 ENV PINPOINT_COLLECTOR_IP=pinpoint-collector @@ -38,12 +40,12 @@ EXPOSE 8080 # pinpoint-root.config에서 Collector IP 오버라이드 (profiler.transport.grpc.collector.ip) # -D 옵션으로 전달 -ENTRYPOINT ["sh", "-c", "java \ +ENTRYPOINT ["sh", "-c", "DEFAULT_JAVA_OPTS=\"-XX:StartFlightRecording=name=dorumdorum,settings=${JFR_SETTINGS:-profile},disk=true,dumponexit=true,filename=/app/logs/jfr/dorumdorum-${JFR_RUN_ID:-cloud-load-test}.jfr,maxage=${JFR_MAX_AGE:-30m},maxsize=${JFR_MAX_SIZE:-512m}\"; EFFECTIVE_JAVA_OPTS=\"${DEFAULT_JAVA_OPTS} ${JAVA_OPTS:-}\"; exec java \ -javaagent:/app/pinpoint-agent/pinpoint-bootstrap-${PINPOINT_VERSION}.jar \ -Dpinpoint.agentId=${PINPOINT_AGENT_ID} \ -Dpinpoint.applicationName=${PINPOINT_APPLICATION_NAME} \ -Dprofiler.transport.grpc.collector.ip=${PINPOINT_COLLECTOR_IP} \ -Dpinpoint.container=true \ - ${JAVA_OPTS} \ + ${EFFECTIVE_JAVA_OPTS} \ -Dspring.profiles.active=${SPRING_PROFILES_ACTIVE:-prod} \ -jar /app/app.jar"] diff --git a/build.gradle b/build.gradle index d4ff20ba..574ce83b 100644 --- a/build.gradle +++ b/build.gradle @@ -94,6 +94,10 @@ dependencies { tasks.named('test') { useJUnitPlatform() + + if (project.hasProperty('skipIntegrationTests')) { + exclude '**/*IntegrationTest.class' + } } def generated = "$buildDir/generated/sources/annotationProcessor/java/main" diff --git a/docker-compose.yml b/docker-compose.yml index bf88b1fe..ecb6ada4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,16 +38,19 @@ services: - "8080:8080" stop_grace_period: 45s env_file: - - .env + - ./secrets/.env environment: - REDIS_HOST=redis - REDIS_PORT=6379 - PINPOINT_COLLECTOR_IP=pinpoint-collector - PINPOINT_APPLICATION_NAME=dorumdorum - - PINPOINT_AGENT_ID=${PINPOINT_AGENT_ID:-dorumdorum-backend-${HOSTNAME}} + - PINPOINT_AGENT_ID=${PINPOINT_AGENT_ID:-dorumdorum-backend-local} - PINPOINT_AGENT_NAME=${PINPOINT_AGENT_NAME:-dorumdorum-be} + - JAVA_OPTS=-Xms6g -Xmx6g -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log -XX:+ExitOnOutOfMemoryError -Xlog:gc*,safepoint:file=/var/log/gc.log:time,uptime,level,tags:filecount=10,filesize=50m volumes: - - ./firebase-service-account.json:/app/firebase-service-account.json:ro + - ./secrets/firebase-service-account.json:/app/firebase-service-account.json:ro + - gc_logs:/var/log + - ./logs/jfr:/app/logs/jfr depends_on: postgres: condition: service_healthy @@ -55,11 +58,8 @@ services: condition: service_started pinpoint-collector: condition: service_started - deploy: - resources: - limits: - cpus: "4.0" - memory: 16G + mem_limit: 8G + cpus: 2.0 restart: unless-stopped networks: - dorumdorum-net @@ -122,7 +122,7 @@ services: image: pinpointdocker/pinpoint-web:2.5.4 container_name: pinpoint-web env_file: - - .env + - ./secrets/.env depends_on: - pinpoint-hbase networks: @@ -144,7 +144,7 @@ services: image: prom/prometheus:v2.52.0 container_name: dorumdorum-prometheus ports: - - "9090:9090" + - "127.0.0.1:9090:9090" volumes: - ./monitoring/prometheus/prometheus.prod.yml:/etc/prometheus/prometheus.yml:ro - prometheus_data:/prometheus @@ -152,12 +152,38 @@ services: - "--config.file=/etc/prometheus/prometheus.yml" - "--storage.tsdb.path=/prometheus" - "--web.enable-lifecycle" + - "--web.enable-remote-write-receiver" extra_hosts: - "host.docker.internal:host-gateway" restart: unless-stopped networks: - dorumdorum-net + k6: + image: ${K6_IMAGE:-grafana/k6:latest} + container_name: dorumdorum-k6 + profiles: + - loadtest + environment: + - BASE_URL=${K6_BASE_URL:-http://backend:8080} + - K6_OUT=${K6_OUT:-experimental-prometheus-rw} + - K6_PROMETHEUS_RW_SERVER_URL=${K6_PROMETHEUS_RW_SERVER_URL:-http://prometheus:9090/api/v1/write} + - K6_PROMETHEUS_RW_PUSH_INTERVAL=${K6_PROMETHEUS_RW_PUSH_INTERVAL:-5s} + - K6_PROMETHEUS_RW_TREND_STATS=${K6_PROMETHEUS_RW_TREND_STATS:-p(90),p(95),p(99),min,max,avg} + - K6_PROMETHEUS_RW_STALE_MARKERS=${K6_PROMETHEUS_RW_STALE_MARKERS:-true} + - K6_WEB_DASHBOARD=${K6_WEB_DASHBOARD:-true} + - K6_WEB_DASHBOARD_EXPORT=${K6_WEB_DASHBOARD_EXPORT:-/results/k6-report.html} + volumes: + - ./load-testing/k6:/scripts:ro + - ./load-testing/results:/results + working_dir: /scripts + entrypoint: ["k6"] + depends_on: + - backend + - prometheus + networks: + - dorumdorum-net + grafana: image: grafana/grafana:11.2.0 container_name: dorumdorum-grafana @@ -185,6 +211,7 @@ volumes: grafana_data: hbase_data: zookeeper_data: + gc_logs: networks: dorumdorum-net: diff --git a/load-testing/README.md b/load-testing/README.md new file mode 100644 index 00000000..026fd456 --- /dev/null +++ b/load-testing/README.md @@ -0,0 +1,16 @@ +# k6 Load Testing + +This directory is reserved for k6 load-test assets. Test scripts are intentionally not included. + +Place scripts under `load-testing/k6/` when needed, then run: + +```sh +docker compose --profile loadtest run --rm k6 run --tag testid=gc-baseline /scripts/