From 0feaa6b23bc3495cec0df184551c5f2e3ef13355 Mon Sep 17 00:00:00 2001 From: Tian Chen <38001883+Tian-2017@users.noreply.github.com> Date: Thu, 4 Dec 2025 10:07:59 +0000 Subject: [PATCH] fix/derive the department name from the s3 path for csv to catalog --- lambdas/csv_to_glue_catalog/main.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lambdas/csv_to_glue_catalog/main.py b/lambdas/csv_to_glue_catalog/main.py index a757162c8..34e95f1fa 100644 --- a/lambdas/csv_to_glue_catalog/main.py +++ b/lambdas/csv_to_glue_catalog/main.py @@ -8,7 +8,6 @@ import logging import re import unicodedata -from os import getenv from pathlib import PurePosixPath from typing import Any from urllib.parse import unquote_plus @@ -283,7 +282,6 @@ def handle_sqs_event(event: dict[str, Any]) -> dict[str, Any]: Returns: Dictionary with batchItemFailures for partial batch failure handling """ - database_name = getenv("GLUE_DATABASE_NAME", "parking_user_uploads_db") sqs_records = event.get("Records", []) total_records = len(sqs_records) processed_count = 0 @@ -309,6 +307,14 @@ def handle_sqs_event(event: dict[str, Any]) -> dict[str, Any]: ) logger.info(f"Processing file from message {message_id}: {s3_key}") + # Extract and normalize department from S3 path to construct database name + department, _, _ = parse_s3_key(s3_key) + normalized_dept = department.replace("-", "_") + database_name = f"{normalized_dept}_user_uploads_db" + logger.info( + f"Using database '{database_name}' for department '{department}'" + ) + was_processed, was_skipped = process_single_event_record( s3_event_record, database_name )