Feature cloudsql sftp nuans (#1933)

jobs/sftp-nuans-report/.env.sample

@@ -1,9 +1,10 @@
 # NameX DB
-PG_USER=
-PG_PASSWORD=
-PG_DB_NAME=
-PG_HOST=
-PG_PORT=
+DATABASE_NAME=
+DATABASE_USERNAME=
+DATABASE_INSTANCE_CONNECTION_NAME=
+DATABASE_SCHEMA=
+DATABASE_IP_TYPE=public
+
 
 # Token Retrieval
 KEYCLOAK_AUTH_TOKEN_URL=

jobs/sftp-nuans-report/Dockerfile

@@ -1,5 +1,4 @@
-FROM python:3.12.2
-USER root
+FROM python:3.12-slim AS development_build
 
 ARG VCS_REF="missing"
 ARG BUILD_DATE="missing"
@@ -8,7 +7,8 @@ ENV VCS_REF=${VCS_REF}
 ENV BUILD_DATE=${BUILD_DATE}
 
 LABEL org.label-schema.vcs-ref=${VCS_REF} \
-  org.label-schema.build-date=${BUILD_DATE}
+      org.label-schema.build-date=${BUILD_DATE} \
+      vendor="BCROS"
 
 USER root
 
@@ -29,7 +29,7 @@ ENV APP_ENV=${APP_ENV} \
   PIP_DEFAULT_TIMEOUT=100 \
   PIP_ROOT_USER_ACTION=ignore \
   # poetry:
-  POETRY_VERSION=1.3.2 \
+  POETRY_VERSION=2.1.3 \
   POETRY_NO_INTERACTION=1 \
   POETRY_VIRTUALENVS_CREATE=false \
   POETRY_CACHE_DIR='/var/cache/pypoetry' \
@@ -67,6 +67,7 @@ RUN chmod 755 /code/run.sh
 RUN --mount=type=cache,target="$POETRY_CACHE_DIR" \
   echo "$APP_ENV" \
   && poetry version \
+  && poetry config installer.max-workers 1 \
   # Install deps:
   && poetry run pip install -U pip \
   && poetry install \
@@ -76,8 +77,8 @@ RUN --mount=type=cache,target="$POETRY_CACHE_DIR" \
 # Running as non-root user:
 USER web
 
-# The following stage is only for production:
-# FROM development_build AS production_build
+# Stage 2: Production image (lighter)
+FROM development_build AS production_build
 COPY --chown=web:web . /code
 RUN chmod -R 755 /code/run.sh
 

jobs/sftp-nuans-report/Makefile

@@ -81,6 +81,14 @@ mac-cov: test ## Run the coverage report and display in a browser window (mac)
 run: ## Run the project in local
 	. .venv/bin/activate && python sftp_nuans_report/sftpnuans.py
 
+#################################################################################
+# Build Docker                                                         #
+#################################################################################
+build: ## Build the Docker container
+	docker build . -t $(DOCKER_NAME) \
+		--build-arg VCS_REF=$(shell git rev-parse --short HEAD) \
+		--build-arg BUILD_DATE=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
+
 #################################################################################
 # Self Documenting Commands                                                     #
 #################################################################################

jobs/sftp-nuans-report/devops/vaults.gcp.env

@@ -1,13 +1,13 @@
-PG_UNIX_SOCKET="op://database/$APP_ENV/namex-db-gcp/DATABASE_UNIX_SOCKET"
-PG_DB_NAME="op://database/$APP_ENV/namex-db-gcp/DATABASE_NAME"
-PG_PASSWORD="op://database/$APP_ENV/namex-db-gcp/DATABASE_PASSWORD"
-PG_PORT="op://database/$APP_ENV/namex-db-gcp/DATABASE_PORT"
-PG_USER="op://database/$APP_ENV/namex-db-gcp/DATABASE_USERNAME"
+DATABASE_NAME="op://database/$APP_ENV/namex-db-gcp/DATABASE_NAME"
+DATABASE_USERNAME="op://database/$APP_ENV/namex-db-gcp/DATABASE_USERNAME"
+DATABASE_INSTANCE_CONNECTION_NAME="op://database/$APP_ENV/namex-db-gcp/DATABASE_INSTANCE_CONNECTION_NAME"
+DATABASE_SCHEMA="op://database/$APP_ENV/namex-db-gcp/DATABASE_SCHEMA"
 ENVIRONMENT="op://namex/$APP_ENV/sftp-nuans-report/ENVIRONMENT"
 ERROR_EMAIL_RECIPIENTS="op://namex/$APP_ENV/sftp-nuans-report/ERROR_EMAIL_RECIPIENTS"
 OCP_SFTP_RELAY_URL="op://namex/$APP_ENV/sftp-nuans-report/OCP_SFTP_RELAY_URL"
 NOTIFY_API_URL="op://API/$APP_ENV/notify-api/NOTIFY_API_URL"
 NOTIFY_API_VERSION="op://API/$APP_ENV/notify-api/NOTIFY_API_VERSION"
 KEYCLOAK_AUTH_TOKEN_URL="op://keycloak/$APP_ENV/base/KEYCLOAK_AUTH_TOKEN_URL"
 KEYCLOAK_CLIENT_ID="op://keycloak/$APP_ENV/entity-service-account/ENTITY_SERVICE_ACCOUNT_CLIENT_ID"
-KEYCLOAK_CLIENT_SECRET="op://keycloak/$APP_ENV/entity-service-account/ENTITY_SERVICE_ACCOUNT_CLIENT_SECRET"
+KEYCLOAK_CLIENT_SECRET="op://keycloak/$APP_ENV/entity-service-account/ENTITY_SERVICE_ACCOUNT_CLIENT_SECRET"
+VPC_CONNECTOR="op://CD/$APP_ENV/base/VPC_CONNECTOR"

jobs/sftp-nuans-report/pyproject.toml

@@ -10,10 +10,8 @@ packages = [{include = "sftp_nuans_report"}]
 python = ">=3.12,<3.13"
 structured_logging = { git = "https://github.com/bcgov/sbc-connect-common.git", subdirectory = "python/structured-logging" }
 registry_schemas = { git = "https://github.com/bcgov/business-schemas.git", rev = "2.5.0" }
+cloud-sql-connector = { git = "https://github.com/bcgov/sbc-connect-common.git", subdirectory = "python/cloud-sql-connector", branch = "main" }
 jupyter = "^1.1.1"
-sqlalchemy = "^2.0.38"
-psycopg2-binary = "^2.9.10"
-ipython-sql = "^0.5.0"
 simplejson = "^3.20.1"
 pandas = "^2.2.3"
 matplotlib = "^3.10.1"

jobs/sftp-nuans-report/run.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 echo 'run sftpnuans'
-python sftp_nuans_report/sftpnuans.py
+python3.12 sftp_nuans_report/sftpnuans.py

jobs/sftp-nuans-report/sftp_nuans_report/config.py

@@ -1,5 +1,6 @@
 import os
-from dotenv import load_dotenv, find_dotenv
+
+from dotenv import find_dotenv, load_dotenv
 
 # this will load all the envars from a .env file located in the project root (api)
 load_dotenv(find_dotenv())
@@ -19,12 +20,17 @@ class Config(object):
     OCP_SFTP_URL = f"{os.getenv("OCP_SFTP_RELAY_URL", "")}/sftp/upload"
 
     # POSTGRESQL
-    PG_USER = os.getenv('PG_USER', '')
-    PG_PASSWORD = os.getenv('PG_PASSWORD', '')
-    PG_NAME = os.getenv('PG_DB_NAME', '')
-    PG_HOST = os.getenv('PG_HOST', '')
-    PG_PORT = os.getenv('PG_PORT', '5432')
-    if DB_UNIX_SOCKET := os.getenv('PG_UNIX_SOCKET', None):
-        SQLALCHEMY_DATABASE_URI = f'postgresql+psycopg2://{PG_USER}:{PG_PASSWORD}@/{PG_NAME}?host={DB_UNIX_SOCKET}'
+    DATABASE_USERNAME = os.getenv('DATABASE_USERNAME', 'postgres')
+    DATABASE_PASSWORD = os.getenv('DATABASE_PASSWORD', 'postgres')
+    DATABASE_NAME = os.getenv('DATABASE_NAME', 'unittesting')
+    DATABASE_HOST = os.getenv('DATABASE_HOST', 'localhost')
+    DATABASE_PORT = os.getenv('DATABASE_PORT', '5432')
+
+    DATABASE_SCHEMA = os.getenv('DATABASE_SCHEMA', 'public')
+    DATABASE_IP_TYPE = os.getenv('DATABASE_IP_TYPE', 'private')
+    DATABASE_OWNER = os.getenv('DATABASE_OWNER', 'postgres')
+
+    if DATABASE_INSTANCE_CONNECTION_NAME := os.getenv('DATABASE_INSTANCE_CONNECTION_NAME', None):
+        SQLALCHEMY_DATABASE_URI = 'postgresql+pg8000://'
     else:
-        SQLALCHEMY_DATABASE_URI = f'postgresql://{PG_USER}:{PG_PASSWORD}@{PG_HOST}:{int(PG_PORT)}/{PG_NAME}'
+        SQLALCHEMY_DATABASE_URI = f'postgresql+pg8000://{DATABASE_USERNAME}:{DATABASE_PASSWORD}@{DATABASE_HOST}:{DATABASE_PORT}/{DATABASE_NAME}'