Feature nr day job db update

api/config.py

@@ -102,7 +102,6 @@ class Config(object):
 
     DISABLE_NAMEREQUEST_SOLR_UPDATES = int(os.getenv('DISABLE_NAMEREQUEST_SOLR_UPDATES', 0))
 
-    GCP_AUTH_KEY = os.getenv('BUSINESS_GCP_AUTH_KEY', None)
     NAMEX_NR_STATE_TOPIC = os.getenv('NAMEX_NR_STATE_TOPIC', '')
     EMAILER_TOPIC = os.getenv('NAMEX_MAILER_TOPIC', '')
 

jobs/nr-day-job/Dockerfile

@@ -1,5 +1,4 @@
-FROM python:3.12.2
-USER root
+FROM python:3.12-slim AS development_build
 
 ARG VCS_REF="missing"
 ARG BUILD_DATE="missing"
@@ -8,7 +7,8 @@ ENV VCS_REF=${VCS_REF}
 ENV BUILD_DATE=${BUILD_DATE}
 
 LABEL org.label-schema.vcs-ref=${VCS_REF} \
-  org.label-schema.build-date=${BUILD_DATE}
+      org.label-schema.build-date=${BUILD_DATE} \
+      vendor="BCROS"
 
 USER root
 
@@ -29,7 +29,7 @@ ENV APP_ENV=${APP_ENV} \
   PIP_DEFAULT_TIMEOUT=100 \
   PIP_ROOT_USER_ACTION=ignore \
   # poetry:
-  POETRY_VERSION=1.3.2 \
+  POETRY_VERSION=2.1.3 \
   POETRY_NO_INTERACTION=1 \
   POETRY_VIRTUALENVS_CREATE=false \
   POETRY_CACHE_DIR='/var/cache/pypoetry' \
@@ -67,6 +67,7 @@ RUN chmod 755 /code/run.sh
 RUN --mount=type=cache,target="$POETRY_CACHE_DIR" \
   echo "$APP_ENV" \
   && poetry version \
+  && poetry config installer.max-workers 1 \
   # Install deps:
   && poetry run pip install -U pip \
   && poetry install \
@@ -76,8 +77,8 @@ RUN --mount=type=cache,target="$POETRY_CACHE_DIR" \
 # Running as non-root user:
 USER web
 
-# The following stage is only for production:
-# FROM development_build AS production_build
+# Stage 2: Production image (lighter)
+FROM development_build AS production_build
 COPY --chown=web:web . /code
 RUN chmod -R 755 /code/run.sh
 

jobs/nr-day-job/Makefile

@@ -50,21 +50,19 @@ install: clean ## Install python virtrual environment
 #################################################################################
 # COMMANDS - CI                                                                 #
 #################################################################################
-ci: lint flake8 test ## CI flow
+ci: docker-build-check ruff test ## CI flow
 
-pylint: ## Linting with pylint
-	. .venv/bin/activate && pylint --rcfile=setup.cfg nr_day_job.py
+docker-build-check: ## Check if Dockerfile builds successfully
+	docker build -f Dockerfile -t nr-day-job .
 
-flake8: ## Linting with flake8
-	. .venv/bin/activate && flake8 nr_day_job.py
+ruff: ## ruff linter
+	poetry run ruff check
 
-lint: pylint flake8 ## run all lint type scripts
+ruff-fix: ## auto fix lint issues with ruff
+	poetry run ruff check --fix
 
-test: ## Unit testing
-	. .venv/bin/activate && pytest
-
-mac-cov: test ## Run the coverage report and display in a browser window (mac)
-	@open -a "Google Chrome" htmlcov/index.html
+test: ## unit testing with local db
+	poetry run pytest
 
 #################################################################################
 # COMMANDS - Local                                                              #

jobs/nr-day-job/config.py

@@ -61,19 +61,20 @@ class Config():  # pylint: disable=too-few-public-methods
     SQLALCHEMY_TRACK_MODIFICATIONS = False
 
     # POSTGRESQL
-    DB_USER = os.getenv('DATABASE_USERNAME', '')
-    DB_PASSWORD = os.getenv('DATABASE_PASSWORD', '')
-    DB_NAME = os.getenv('DATABASE_NAME', '')
-    DB_HOST = os.getenv('DATABASE_HOST', '')
+    DB_USER = os.getenv('DATABASE_USERNAME', 'postgres')
+    DB_PASSWORD = os.getenv('DATABASE_PASSWORD', 'postgres')
+    DB_NAME = os.getenv('DATABASE_NAME', 'unittesting')
+    DB_HOST = os.getenv('DATABASE_HOST', 'localhost')
     DB_PORT = os.getenv('DATABASE_PORT', '5432')
-    if DB_UNIX_SOCKET := os.getenv('DATABASE_UNIX_SOCKET', None):
-        SQLALCHEMY_DATABASE_URI = f'postgresql+psycopg2://{DB_USER}:{DB_PASSWORD}@/{DB_NAME}?host={DB_UNIX_SOCKET}'
-    else:
-        SQLALCHEMY_DATABASE_URI = f'postgresql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}'
 
-    # Normalize the b64 key: strip whitespace and add padding as needed.
-    _key = os.getenv('BUSINESS_GCP_AUTH_KEY', None)
-    GCP_AUTH_KEY = None if _key is None else "".join(_key.split()) + "=" * (-len("".join(_key.split())) % 4)
+    DB_SCHEMA = os.getenv('DATABASE_SCHEMA', 'public')
+    DB_IP_TYPE = os.getenv('DATABASE_IP_TYPE', 'private')
+    DB_OWNER = os.getenv('DATABASE_OWNER', 'postgres')
+
+    if DB_INSTANCE_CONNECTION_NAME := os.getenv('DATABASE_INSTANCE_CONNECTION_NAME', None):
+        SQLALCHEMY_DATABASE_URI = 'postgresql+pg8000://'
+    else:
+        SQLALCHEMY_DATABASE_URI = f'postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}'
 
     EMAILER_TOPIC = os.getenv('NAMEX_MAILER_TOPIC', '')
     NAMEX_NR_STATE_TOPIC = os.getenv('NAMEX_NR_STATE_TOPIC', '')
@@ -93,20 +94,14 @@ class TestConfig(Config):  # pylint: disable=too-few-public-methods
 
     DEBUG = True
     TESTING = True
+
     # POSTGRESQL
-    DB_USER = os.getenv('DATABASE_TEST_USERNAME', '')
-    DB_PASSWORD = os.getenv('DATABASE_TEST_PASSWORD', '')
-    DB_NAME = os.getenv('DATABASE_TEST_NAME', '')
-    DB_HOST = os.getenv('DATABASE_TEST_HOST', '')
+    DB_USER = os.getenv('DATABASE_TEST_USERNAME', 'postgres')
+    DB_PASSWORD = os.getenv('DATABASE_TEST_PASSWORD', 'postgres')
+    DB_NAME = os.getenv('DATABASE_TEST_NAME', 'unittesting')
+    DB_HOST = os.getenv('DATABASE_TEST_HOST', 'localhost')
     DB_PORT = os.getenv('DATABASE_TEST_PORT', '5432')
-    SQLALCHEMY_DATABASE_URI = 'postgresql://{user}:{password}@{host}:{port}/{name}'.format(
-        user=DB_USER,
-        password=DB_PASSWORD,
-        host=DB_HOST,
-        port=int(DB_PORT),
-        name=DB_NAME,
-    )
-
+    SQLALCHEMY_DATABASE_URI = f'postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}'
 
 class ProdConfig(Config):  # pylint: disable=too-few-public-methods
     """Production environment configuration."""

jobs/nr-day-job/devops/vaults.gcp.env

@@ -1,8 +1,7 @@
 DATABASE_USERNAME="op://database/$APP_ENV/namex-db-gcp/DATABASE_USERNAME"
-DATABASE_PASSWORD="op://database/$APP_ENV/namex-db-gcp/DATABASE_PASSWORD"
 DATABASE_NAME="op://database/$APP_ENV/namex-db-gcp/DATABASE_NAME"
-DATABASE_UNIX_SOCKET="op://database/$APP_ENV/namex-db-gcp/DATABASE_UNIX_SOCKET"
-DATABASE_PORT="op://database/$APP_ENV/namex-db-gcp/DATABASE_PORT"
-BUSINESS_GCP_AUTH_KEY="op://gcp-queue/$APP_ENV/a083gt/BUSINESS_GCP_AUTH_KEY"
+DATABASE_INSTANCE_CONNECTION_NAME="op://database/$APP_ENV/namex-db-gcp/DATABASE_INSTANCE_CONNECTION_NAME"
+DATABASE_SCHEMA="op://database/$APP_ENV/namex-db-gcp/DATABASE_SCHEMA"
 NAMEX_MAILER_TOPIC="op://gcp-queue/$APP_ENV/topics/NAMEX_MAILER_TOPIC"
-NAMEX_NR_STATE_TOPIC="op://gcp-queue/$APP_ENV/topics/NAMEX_NR_STATE_TOPIC"
+NAMEX_NR_STATE_TOPIC="op://gcp-queue/$APP_ENV/topics/NAMEX_NR_STATE_TOPIC"
+VPC_CONNECTOR="op://CD/$APP_ENV/base/VPC_CONNECTOR"

jobs/nr-day-job/nr_day_job.py

@@ -17,20 +17,22 @@
 from datetime import datetime, timezone
 
 from flask import Flask, current_app
-from namex.models import Request, State, db, Event
-from namex.services import queue, EventRecorder
+from namex import DBConfig, setup_search_path_event_listener
+from namex.models import Event, Request, State, db
+from namex.services import EventRecorder, queue
 from sbc_common_components.utils.enums import QueueMessageTypes
 from simple_cloudevent import SimpleCloudEvent
 from sqlalchemy import text
 from structured_logging import StructuredLogging
 
-import config
+from config import Config, ProdConfig, get_named_config
 
-APP_CONFIG = config.get_named_config(os.getenv('FLASK_ENV', 'production'))
+APP_CONFIG = get_named_config(os.getenv('FLASK_ENV', 'production'))
 
 
-def create_app():
+def create_app(environment: Config = ProdConfig, **kwargs) -> Flask:
     """Return a configured Flask App using the Factory method."""
+
     app = Flask(__name__)
     app.config.from_object(APP_CONFIG)
 
@@ -39,9 +41,27 @@ def create_app():
     structured_logger.init_app(app)
     app.logger = structured_logger.get_logger()
 
-    queue.init_app(app)
+    schema = app.config.get('DB_SCHEMA', 'public')
+
+    if app.config.get('DB_INSTANCE_CONNECTION_NAME'):
+        db_config = DBConfig(
+            instance_name=app.config.get('DB_INSTANCE_CONNECTION_NAME'),
+            database=app.config.get('DB_NAME'),
+            user=app.config.get('DB_USER'),
+            ip_type=app.config.get('DB_IP_TYPE'),
+            schema=schema,
+            pool_recycle=300,
+        )
+        app.config['SQLALCHEMY_ENGINE_OPTIONS'] = db_config.get_engine_options()
+
     db.init_app(app)
 
+    if app.config.get('DB_INSTANCE_CONNECTION_NAME'):
+        with app.app_context():
+            engine = db.engine
+            setup_search_path_event_listener(engine, schema)
+
+    queue.init_app(app)
     register_shellcontext(app)
 
     return app
@@ -71,7 +91,7 @@ def furnish_request_message(
     ce = SimpleCloudEvent(
         id=str(uuid.uuid4()),
         source=f'/requests/{request.nrNum}',
-        subject="namerequest",
+        subject='namerequest',
         type=QueueMessageTypes.NAMES_MESSAGE_TYPE.value,
         time=datetime.now(tz=timezone.utc).isoformat(),
         data={