From c75a45cfb93188a63a54cafbb4b3d3a87507c28c Mon Sep 17 00:00:00 2001 From: Andrew Nguyen Date: Mon, 27 Oct 2025 01:14:29 -0700 Subject: [PATCH 1/8] New data format to praises --- api/messages/messages_service.py | 34 ++++++++++++++++++++++++++++---- api/messages/messages_views.py | 4 ++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/api/messages/messages_service.py b/api/messages/messages_service.py index 5f30657..f0d3441 100644 --- a/api/messages/messages_service.py +++ b/api/messages/messages_service.py @@ -1766,10 +1766,8 @@ def save_news(json): return Message("Saved News") def save_praise(json): - logger.debug(f"Attempting to save the praise with the json object {json}") - - # Make sure these fields exist praise_receiver, praise_channel, praise_message - check_fields = ["praise_receiver", "praise_channel", "praise_message"] + # Make sure these fields exist praise_receiver, praise_sender, praise_channel, praise_message + check_fields = ["praise_receiver", "praise_sender", "praise_channel", "praise_message"] for field in check_fields: if field not in json: logger.error(f"Missing field {field} in {json}") @@ -1777,6 +1775,34 @@ def save_praise(json): logger.debug(f"Detected required fields, attempting to save praise") json["timestamp"] = datetime.now(pytz.utc).astimezone().isoformat() + + # Add ohack.dev user IDs for both sender and receiver + try: + # Get ohack.dev user ID for praise receiver + receiver_user = get_user_by_user_id(json["praise_receiver"]) + if receiver_user and "id" in receiver_user: + json["praise_receiver_ohack_id"] = receiver_user["id"] + logger.debug(f"Added praise_receiver_ohack_id: {receiver_user['id']}") + else: + logger.warning(f"Could not find ohack.dev user for praise_receiver: {json['praise_receiver']}") + json["praise_receiver_ohack_id"] = None + + # Get ohack.dev user ID for praise sender + sender_user = get_user_by_user_id(json["praise_sender"]) + if sender_user and "id" in sender_user: + json["praise_sender_ohack_id"] = sender_user["id"] + logger.debug(f"Added praise_sender_ohack_id: {sender_user['id']}") + else: + logger.warning(f"Could not find ohack.dev user for praise_sender: {json['praise_sender']}") + json["praise_sender_ohack_id"] = None + + except Exception as e: + logger.error(f"Error getting ohack.dev user IDs: {str(e)}") + json["praise_receiver_ohack_id"] = None + json["praise_sender_ohack_id"] = None + + + logger.info(f"Attempting to save the praise with the json object {json}") upsert_praise(json) logger.info("Updated praise successfully") diff --git a/api/messages/messages_views.py b/api/messages/messages_views.py index b385c57..ed73bd6 100644 --- a/api/messages/messages_views.py +++ b/api/messages/messages_views.py @@ -505,7 +505,11 @@ def store_praise(): receiver_id = json_data.get("praise_receiver") # Check BACKEND_NEWS_TOKEN + logger.info(f"Token: {token}") + environment_token = os.getenv("BACKEND_PRAISE_TOKEN") + logger.info(f"environment praise token: {environment_token}") if token == None or token != os.getenv("BACKEND_PRAISE_TOKEN"): + logger.info(f"Token mismatch: {token} != {environment_token}") return "Unauthorized", 401 elif sender_id == receiver_id: return "You cannot write a praise about yourself", 400 From 026e7867a93a1c15e290df2a664dcb3949fc28af Mon Sep 17 00:00:00 2001 From: Andrew Nguyen Date: Mon, 27 Oct 2025 01:20:27 -0700 Subject: [PATCH 2/8] Removed unnecessary logs and changes --- api/messages/messages_service.py | 2 +- api/messages/messages_views.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/api/messages/messages_service.py b/api/messages/messages_service.py index f0d3441..14d6b88 100644 --- a/api/messages/messages_service.py +++ b/api/messages/messages_service.py @@ -1767,7 +1767,7 @@ def save_news(json): def save_praise(json): # Make sure these fields exist praise_receiver, praise_sender, praise_channel, praise_message - check_fields = ["praise_receiver", "praise_sender", "praise_channel", "praise_message"] + check_fields = ["praise_receiver", "praise_channel", "praise_message"] for field in check_fields: if field not in json: logger.error(f"Missing field {field} in {json}") diff --git a/api/messages/messages_views.py b/api/messages/messages_views.py index ed73bd6..b385c57 100644 --- a/api/messages/messages_views.py +++ b/api/messages/messages_views.py @@ -505,11 +505,7 @@ def store_praise(): receiver_id = json_data.get("praise_receiver") # Check BACKEND_NEWS_TOKEN - logger.info(f"Token: {token}") - environment_token = os.getenv("BACKEND_PRAISE_TOKEN") - logger.info(f"environment praise token: {environment_token}") if token == None or token != os.getenv("BACKEND_PRAISE_TOKEN"): - logger.info(f"Token mismatch: {token} != {environment_token}") return "Unauthorized", 401 elif sender_id == receiver_id: return "You cannot write a praise about yourself", 400 From 3c7b7d56f7a8bd4fad163e76be112ea10f05700d Mon Sep 17 00:00:00 2001 From: Andrew Nguyen Date: Mon, 27 Oct 2025 01:22:04 -0700 Subject: [PATCH 3/8] edited misleading comment --- api/messages/messages_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/messages/messages_service.py b/api/messages/messages_service.py index 14d6b88..eecf483 100644 --- a/api/messages/messages_service.py +++ b/api/messages/messages_service.py @@ -1766,7 +1766,7 @@ def save_news(json): return Message("Saved News") def save_praise(json): - # Make sure these fields exist praise_receiver, praise_sender, praise_channel, praise_message + # Make sure these fields exist praise_receiver, praise_channel, praise_message check_fields = ["praise_receiver", "praise_channel", "praise_message"] for field in check_fields: if field not in json: From e916896ed5a8ed8661c358e2cbdf3cc9ba28be15 Mon Sep 17 00:00:00 2001 From: Greg V Date: Tue, 28 Oct 2025 20:50:09 -0700 Subject: [PATCH 4/8] Update api/messages/messages_service.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- api/messages/messages_service.py | 1 - 1 file changed, 1 deletion(-) diff --git a/api/messages/messages_service.py b/api/messages/messages_service.py index eecf483..d29e566 100644 --- a/api/messages/messages_service.py +++ b/api/messages/messages_service.py @@ -1801,7 +1801,6 @@ def save_praise(json): json["praise_receiver_ohack_id"] = None json["praise_sender_ohack_id"] = None - logger.info(f"Attempting to save the praise with the json object {json}") upsert_praise(json) From 06cb379c8b67beb3c0412a81193936ef3a9aee30 Mon Sep 17 00:00:00 2001 From: Greg V Date: Mon, 24 Nov 2025 11:42:17 -0700 Subject: [PATCH 5/8] Praise Backfill Script ``` % python add_new_praise_columns.py 2025-11-24 11:41:22,313 - INFO - Starting praise column backfill script Missing ENVIRONMENT environment variable. Setting default to CHANGEMEPLS 2025-11-24 11:41:22,313 - WARNING - Missing ENVIRONMENT environment variable. Setting default to CHANGEMEPLS 2025-11-24 11:41:22,313 - INFO - Querying praises collection... 2025-11-24 11:41:27,001 - INFO - Found 27 total praises 2025-11-24 11:41:27,001 - INFO - Pass 1: Analyzing praises and looking up users... /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages/google/cloud/firestore_v1/base_collection.py:295: UserWarning: Detected filter using positional arguments. Prefer using the 'filter' keyword argument instead. return query.where(field_path, op_string, value) 2025-11-24 11:41:36,530 - INFO - Pass 1 complete: 27 praises need updating 2025-11-24 11:41:36,530 - INFO - User cache contains 12 unique users 2025-11-24 11:41:36,530 - INFO - Pass 2: Applying updates in batches... 2025-11-24 11:41:36,530 - INFO - Processing batch 1/1 (27 updates)... 2025-11-24 11:41:40,917 - INFO - Successfully committed batch 1 2025-11-24 11:41:40,917 - INFO - ================================================================================ 2025-11-24 11:41:40,917 - INFO - Backfill complete! 2025-11-24 11:41:40,917 - INFO - Total praises processed: 27 2025-11-24 11:41:40,917 - INFO - Successfully updated: 27 2025-11-24 11:41:40,917 - INFO - Skipped (already complete): 0 2025-11-24 11:41:40,917 - INFO - Errors encountered: 0 2025-11-24 11:41:40,917 - INFO - Unique users cached: 12 2025-11-24 11:41:40,917 - INFO - ================================================================================ ``` --- scripts/add_new_praise_columns.py | 215 ++++++++++++++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100755 scripts/add_new_praise_columns.py diff --git a/scripts/add_new_praise_columns.py b/scripts/add_new_praise_columns.py new file mode 100755 index 0000000..02304da --- /dev/null +++ b/scripts/add_new_praise_columns.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +""" +Backfill script to populate praise_receiver_ohack_id and praise_sender_ohack_id +in the praises collection. + +This script queries the praises collection, finds records missing these IDs, +and looks up the corresponding user records using their Slack IDs. +""" + +import sys +import os + +from dotenv import load_dotenv +load_dotenv() + +# Add parent directory to path to import from project modules +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from common.utils.firebase import get_db +import logging + +# Set up logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + +SLACK_PREFIX = "oauth2|slack|T1Q7936BH-" + + +def fetch_user_by_slack_id(db, slack_user_id, cache=None): + """ + Fetch a user from the users collection by their Slack user ID. + Uses an optional cache to avoid repeated database queries. + + Args: + db: Firestore database client + slack_user_id (str): Slack user ID (without prefix) + cache (dict, optional): Dictionary to cache user lookups + + Returns: + dict: User document with 'id' field, or None if not found + """ + # Add prefix if not present + if not slack_user_id.startswith(SLACK_PREFIX): + full_slack_id = f"{SLACK_PREFIX}{slack_user_id}" + else: + full_slack_id = slack_user_id + + # Check cache first + if cache is not None and full_slack_id in cache: + logger.debug(f"Cache hit for slack_id: {full_slack_id}") + return cache[full_slack_id] + + logger.debug(f"Cache miss - looking up user with slack_id: {full_slack_id}") + + # Query users collection + docs = db.collection('users').where("user_id", "==", full_slack_id).stream() + + for doc in docs: + user_dict = doc.to_dict() + user_dict['id'] = doc.id + logger.debug(f"Found user: {user_dict.get('name', 'Unknown')} (ID: {doc.id})") + + # Store in cache if provided + if cache is not None: + cache[full_slack_id] = user_dict + + return user_dict + + logger.warning(f"User not found for slack_id: {full_slack_id}") + + # Cache the negative result to avoid repeated lookups + if cache is not None: + cache[full_slack_id] = None + + return None + + +def backfill_praise_columns(): + """ + Main function to backfill praise_receiver_ohack_id and praise_sender_ohack_id + for all praises missing these fields. + + Optimizations: + - Uses in-memory cache to avoid duplicate user lookups + - Batches database writes (up to 500 operations per batch) + - Two-pass approach: collect data, then batch update + """ + logger.info("Starting praise column backfill script") + + # Get database connection + db = get_db() + + # Initialize user lookup cache + user_cache = {} + + # Query all praises + logger.info("Querying praises collection...") + praises_ref = db.collection('praises') + all_praises = list(praises_ref.stream()) + + logger.info(f"Found {len(all_praises)} total praises") + + # Counters for statistics + total_count = 0 + updated_count = 0 + skipped_count = 0 + error_count = 0 + cache_hit_count = 0 + cache_miss_count = 0 + + # Collect updates to batch + updates_to_apply = [] + + # First pass: Analyze all praises and prepare updates + logger.info("Pass 1: Analyzing praises and looking up users...") + for praise_doc in all_praises: + total_count += 1 + praise_data = praise_doc.to_dict() + praise_id = praise_doc.id + + if total_count % 50 == 0: + logger.info(f"Processed {total_count}/{len(all_praises)} praises...") + + # Check if both fields are already populated + has_receiver_id = 'praise_receiver_ohack_id' in praise_data and praise_data['praise_receiver_ohack_id'] + has_sender_id = 'praise_sender_ohack_id' in praise_data and praise_data['praise_sender_ohack_id'] + + if has_receiver_id and has_sender_id: + skipped_count += 1 + continue + + # Prepare update data + update_data = {} + + # Look up receiver if needed + if not has_receiver_id: + praise_receiver = praise_data.get('praise_receiver') + if praise_receiver: + receiver_user = fetch_user_by_slack_id(db, praise_receiver, cache=user_cache) + if receiver_user: + update_data['praise_receiver_ohack_id'] = receiver_user['id'] + else: + logger.warning(f" Praise {praise_id}: Could not find receiver user for slack_id: {praise_receiver}") + error_count += 1 + + # Look up sender if needed + if not has_sender_id: + praise_sender = praise_data.get('praise_sender') + if praise_sender: + sender_user = fetch_user_by_slack_id(db, praise_sender, cache=user_cache) + if sender_user: + update_data['praise_sender_ohack_id'] = sender_user['id'] + else: + logger.warning(f" Praise {praise_id}: Could not find sender user for slack_id: {praise_sender}") + error_count += 1 + + # Queue update if we have data to update + if update_data: + updates_to_apply.append({ + 'reference': praise_doc.reference, + 'data': update_data, + 'praise_id': praise_id + }) + + logger.info(f"Pass 1 complete: {len(updates_to_apply)} praises need updating") + logger.info(f"User cache contains {len(user_cache)} unique users") + + # Second pass: Apply updates in batches + if updates_to_apply: + logger.info("Pass 2: Applying updates in batches...") + batch_size = 500 # Firestore limit + num_batches = (len(updates_to_apply) + batch_size - 1) // batch_size + + for batch_num in range(num_batches): + start_idx = batch_num * batch_size + end_idx = min(start_idx + batch_size, len(updates_to_apply)) + batch_updates = updates_to_apply[start_idx:end_idx] + + logger.info(f"Processing batch {batch_num + 1}/{num_batches} ({len(batch_updates)} updates)...") + + # Create Firestore batch + batch = db.batch() + + for update in batch_updates: + batch.update(update['reference'], update['data']) + + # Commit the batch + try: + batch.commit() + updated_count += len(batch_updates) + logger.info(f" Successfully committed batch {batch_num + 1}") + except Exception as e: + logger.error(f" Error committing batch {batch_num + 1}: {str(e)}") + error_count += len(batch_updates) + + # Print summary + logger.info("=" * 80) + logger.info("Backfill complete!") + logger.info(f"Total praises processed: {total_count}") + logger.info(f"Successfully updated: {updated_count}") + logger.info(f"Skipped (already complete): {skipped_count}") + logger.info(f"Errors encountered: {error_count}") + logger.info(f"Unique users cached: {len(user_cache)}") + logger.info("=" * 80) + + +if __name__ == "__main__": + try: + backfill_praise_columns() + except Exception as e: + logger.error(f"Fatal error: {str(e)}", exc_info=True) + sys.exit(1) From b6b300e4db53d531af80646e8dd95ecea7506b21 Mon Sep 17 00:00:00 2001 From: Greg V Date: Mon, 24 Nov 2025 11:45:09 -0700 Subject: [PATCH 6/8] Updates to support sending emails --- api/teams/teams_service.py | 14 ++++++++++++++ services/volunteers_service.py | 11 ++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/api/teams/teams_service.py b/api/teams/teams_service.py index 490452e..03b14a0 100644 --- a/api/teams/teams_service.py +++ b/api/teams/teams_service.py @@ -905,6 +905,20 @@ def send_team_message(admin_user, teamid, json): slack_channel = team_data["slack_channel"] send_slack(message, slack_channel) + # I'd also like to send each of them an email using send_volunteer_message from volunteers_service.py where we need to get their Slack User ID and pass that in as recipient_id + from api.volunteers.volunteers_service import send_volunteer_message, get_user_from_slack_id + for user_ref in team_data.get("users", []): + user_data = user_ref.get().to_dict() + slack_user_id = user_data.get("user_id", "") + if slack_user_id: + send_volunteer_message( + admin_user=admin_user, + recipient_id=slack_user_id, + subject=f"Message from Opportunity Hack Team {team_data['name']}", + message=message + ) + + # If communication_history doesn't exist, create it and append the timestamp, sender, text if "communication_history" not in team_data: team_data["communication_history"] = [] diff --git a/services/volunteers_service.py b/services/volunteers_service.py index e2b37ac..25e759d 100644 --- a/services/volunteers_service.py +++ b/services/volunteers_service.py @@ -1599,8 +1599,17 @@ def send_volunteer_message( volunteer_doc = db.collection('volunteers').document(volunteer_id).get() SLACK_USER_PREFIX = "oauth2|slack|T1Q7936BH-" + + # if recipient_id already has SLACK_USER_PREFIX, use it directly + if recipient_id and recipient_id.startswith(SLACK_USER_PREFIX): + pass + elif recipient_id: + recipient_id = f"{SLACK_USER_PREFIX}{recipient_id}" + else: + logger.warning("No recipient_id provided, Slack message may not be sent if slack_user_id is not found in volunteer record.") + # Search users for "user_id": "" - users_doc = db.collection('users').where('user_id', '==', f"{SLACK_USER_PREFIX}{recipient_id}").get() + users_doc = db.collection('users').where('user_id', '==', f"{recipient_id}").get() email = None slack_user_id = None From d65187d262d2fdf4e1029108ee2aeedab836d5af Mon Sep 17 00:00:00 2001 From: Greg V Date: Wed, 17 Dec 2025 20:38:08 -0700 Subject: [PATCH 7/8] Add colorful and easier to read logs --- api/__init__.py | 57 ++++++++++++++++++++++++++++++++++++------------ common/log.py | 31 +++++++++++++++++++++++++- requirements.txt | 3 ++- 3 files changed, 75 insertions(+), 16 deletions(-) diff --git a/api/__init__.py b/api/__init__.py index bf40b1c..ff76272 100644 --- a/api/__init__.py +++ b/api/__init__.py @@ -11,6 +11,12 @@ from common.utils import safe_get_env_var import os +try: + import colorlog + HAS_COLORLOG = True +except ImportError: + HAS_COLORLOG = False + def grouping_function(request): env = safe_get_env_var("FLASK_ENV") if True: @@ -26,27 +32,50 @@ def grouping_function(request): return None +# Determine if colored logging should be used +use_colored_logs = ( + HAS_COLORLOG and + os.environ.get('USE_COLORED_LOGS', 'true').lower() == 'true' +) + +# Build formatters dictionary +formatters = { + 'default': { + 'format': '[%(asctime)s] {%(pathname)s:%(funcName)s:%(lineno)d} %(levelname)s - %(message)s', + } +} + +if use_colored_logs: + formatters['colored'] = { + '()': 'colorlog.ColoredFormatter', + 'format': '%(log_color)s[%(asctime)s] %(levelname)s%(reset)s - %(message)s', + 'datefmt': '%Y-%m-%d %H:%M:%S', + 'log_colors': { + 'DEBUG': 'cyan', + 'INFO': 'green', + 'WARNING': 'yellow', + 'ERROR': 'red', + 'CRITICAL': 'red,bg_white', + } + } + dict_config = { 'version': 1, - 'formatters': { - 'default': { - 'format': '[%(asctime)s] {%(pathname)s:%(funcName)s:%(lineno)d} %(levelname)s - %(message)s', - } - }, + 'formatters': formatters, 'handlers': { 'default': { - 'level': 'DEBUG', - 'formatter': 'default', - 'class': 'logging.handlers.RotatingFileHandler', - 'filename': "test.log", - 'maxBytes': 5000000, - 'backupCount': 10 - }, + 'level': 'DEBUG', + 'formatter': 'default', + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': "test.log", + 'maxBytes': 5000000, + 'backupCount': 10 + }, 'console': { 'class': 'logging.StreamHandler', 'level': 'DEBUG', - 'formatter': 'default', - }, + 'formatter': 'colored' if use_colored_logs else 'default', + }, }, 'loggers': { 'myapp': { diff --git a/common/log.py b/common/log.py index b072f63..cca2b5a 100644 --- a/common/log.py +++ b/common/log.py @@ -6,6 +6,12 @@ import sys from typing import Any, Dict, Optional, Union +try: + import colorlog + HAS_COLORLOG = True +except ImportError: + HAS_COLORLOG = False + # Configure default log level log_level = logging.INFO @@ -65,6 +71,14 @@ def format(self, record: logging.LogRecord) -> str: # Determine if we should use JSON logging based on environment use_json_logging = os.environ.get('USE_JSON_LOGGING', 'false').lower() == 'true' +# Determine if we should use colored logging (only in non-JSON mode) +# Default to True if colorlog is available and we're not using JSON logging +use_colored_logging = ( + HAS_COLORLOG and + not use_json_logging and + os.environ.get('USE_COLORED_LOGS', 'true').lower() == 'true' +) + # Root logger configuration def configure_root_logger(): """Configure the root logger with appropriate handlers""" @@ -82,9 +96,24 @@ def configure_root_logger(): # Apply appropriate formatter if use_json_logging: formatter = JsonFormatter() + elif use_colored_logging: + # Use colorlog for colored output with nice formatting + formatter = colorlog.ColoredFormatter( + '%(log_color)s%(asctime)s - %(name)s - %(levelname)s%(reset)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + log_colors={ + 'DEBUG': 'cyan', + 'INFO': 'green', + 'WARNING': 'yellow', + 'ERROR': 'red', + 'CRITICAL': 'red,bg_white', + }, + secondary_log_colors={}, + style='%' + ) else: formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - + console_handler.setFormatter(formatter) root_logger.addHandler(console_handler) diff --git a/requirements.txt b/requirements.txt index 0181c77..8610711 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,4 +43,5 @@ resend==2.3.0 readme-metrics[Flask]==3.1.0 redis==5.2.1 tiktoken==0.9.0 -numpy==1.26.3 \ No newline at end of file +numpy==1.26.3 +colorlog==6.7.0 \ No newline at end of file From 347e524658edd8b66e29bb8569b95006f76e3d29 Mon Sep 17 00:00:00 2001 From: Greg V Date: Wed, 17 Dec 2025 20:38:30 -0700 Subject: [PATCH 8/8] [bugfix] handle when we don't have a github repo for a hackathon --- api/leaderboard/leaderboard_service.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/api/leaderboard/leaderboard_service.py b/api/leaderboard/leaderboard_service.py index a203e8d..4102cee 100644 --- a/api/leaderboard/leaderboard_service.py +++ b/api/leaderboard/leaderboard_service.py @@ -543,6 +543,18 @@ def get_github_leaderboard(event_id: str) -> Dict[str, Any]: org_name = get_github_organizations(event_id) org_duration = time.time() - org_start logger.debug("get_github_organizations took %.2f seconds", org_duration) + + if org_name["github_organizations"] == []: + logger.warning("No GitHub organizations found for event ID: %s", event_id) + return { + "github_organizations": [], + "github_repositories": [], + "github_contributors": [], + "github_achievements": [], + "generalStats": [], + "individualAchievements": [], + "teamAchievements": [] + } repos_start = time.time() repos = get_github_repositories(org_name["github_organizations"][0]["name"])