diff --git a/api/__init__.py b/api/__init__.py index bf40b1c..ff76272 100644 --- a/api/__init__.py +++ b/api/__init__.py @@ -11,6 +11,12 @@ from common.utils import safe_get_env_var import os +try: + import colorlog + HAS_COLORLOG = True +except ImportError: + HAS_COLORLOG = False + def grouping_function(request): env = safe_get_env_var("FLASK_ENV") if True: @@ -26,27 +32,50 @@ def grouping_function(request): return None +# Determine if colored logging should be used +use_colored_logs = ( + HAS_COLORLOG and + os.environ.get('USE_COLORED_LOGS', 'true').lower() == 'true' +) + +# Build formatters dictionary +formatters = { + 'default': { + 'format': '[%(asctime)s] {%(pathname)s:%(funcName)s:%(lineno)d} %(levelname)s - %(message)s', + } +} + +if use_colored_logs: + formatters['colored'] = { + '()': 'colorlog.ColoredFormatter', + 'format': '%(log_color)s[%(asctime)s] %(levelname)s%(reset)s - %(message)s', + 'datefmt': '%Y-%m-%d %H:%M:%S', + 'log_colors': { + 'DEBUG': 'cyan', + 'INFO': 'green', + 'WARNING': 'yellow', + 'ERROR': 'red', + 'CRITICAL': 'red,bg_white', + } + } + dict_config = { 'version': 1, - 'formatters': { - 'default': { - 'format': '[%(asctime)s] {%(pathname)s:%(funcName)s:%(lineno)d} %(levelname)s - %(message)s', - } - }, + 'formatters': formatters, 'handlers': { 'default': { - 'level': 'DEBUG', - 'formatter': 'default', - 'class': 'logging.handlers.RotatingFileHandler', - 'filename': "test.log", - 'maxBytes': 5000000, - 'backupCount': 10 - }, + 'level': 'DEBUG', + 'formatter': 'default', + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': "test.log", + 'maxBytes': 5000000, + 'backupCount': 10 + }, 'console': { 'class': 'logging.StreamHandler', 'level': 'DEBUG', - 'formatter': 'default', - }, + 'formatter': 'colored' if use_colored_logs else 'default', + }, }, 'loggers': { 'myapp': { diff --git a/api/leaderboard/leaderboard_service.py b/api/leaderboard/leaderboard_service.py index a203e8d..4102cee 100644 --- a/api/leaderboard/leaderboard_service.py +++ b/api/leaderboard/leaderboard_service.py @@ -543,6 +543,18 @@ def get_github_leaderboard(event_id: str) -> Dict[str, Any]: org_name = get_github_organizations(event_id) org_duration = time.time() - org_start logger.debug("get_github_organizations took %.2f seconds", org_duration) + + if org_name["github_organizations"] == []: + logger.warning("No GitHub organizations found for event ID: %s", event_id) + return { + "github_organizations": [], + "github_repositories": [], + "github_contributors": [], + "github_achievements": [], + "generalStats": [], + "individualAchievements": [], + "teamAchievements": [] + } repos_start = time.time() repos = get_github_repositories(org_name["github_organizations"][0]["name"]) diff --git a/api/messages/messages_service.py b/api/messages/messages_service.py index 5f30657..d29e566 100644 --- a/api/messages/messages_service.py +++ b/api/messages/messages_service.py @@ -1766,8 +1766,6 @@ def save_news(json): return Message("Saved News") def save_praise(json): - logger.debug(f"Attempting to save the praise with the json object {json}") - # Make sure these fields exist praise_receiver, praise_channel, praise_message check_fields = ["praise_receiver", "praise_channel", "praise_message"] for field in check_fields: @@ -1777,6 +1775,33 @@ def save_praise(json): logger.debug(f"Detected required fields, attempting to save praise") json["timestamp"] = datetime.now(pytz.utc).astimezone().isoformat() + + # Add ohack.dev user IDs for both sender and receiver + try: + # Get ohack.dev user ID for praise receiver + receiver_user = get_user_by_user_id(json["praise_receiver"]) + if receiver_user and "id" in receiver_user: + json["praise_receiver_ohack_id"] = receiver_user["id"] + logger.debug(f"Added praise_receiver_ohack_id: {receiver_user['id']}") + else: + logger.warning(f"Could not find ohack.dev user for praise_receiver: {json['praise_receiver']}") + json["praise_receiver_ohack_id"] = None + + # Get ohack.dev user ID for praise sender + sender_user = get_user_by_user_id(json["praise_sender"]) + if sender_user and "id" in sender_user: + json["praise_sender_ohack_id"] = sender_user["id"] + logger.debug(f"Added praise_sender_ohack_id: {sender_user['id']}") + else: + logger.warning(f"Could not find ohack.dev user for praise_sender: {json['praise_sender']}") + json["praise_sender_ohack_id"] = None + + except Exception as e: + logger.error(f"Error getting ohack.dev user IDs: {str(e)}") + json["praise_receiver_ohack_id"] = None + json["praise_sender_ohack_id"] = None + + logger.info(f"Attempting to save the praise with the json object {json}") upsert_praise(json) logger.info("Updated praise successfully") diff --git a/api/teams/teams_service.py b/api/teams/teams_service.py index 490452e..03b14a0 100644 --- a/api/teams/teams_service.py +++ b/api/teams/teams_service.py @@ -905,6 +905,20 @@ def send_team_message(admin_user, teamid, json): slack_channel = team_data["slack_channel"] send_slack(message, slack_channel) + # I'd also like to send each of them an email using send_volunteer_message from volunteers_service.py where we need to get their Slack User ID and pass that in as recipient_id + from api.volunteers.volunteers_service import send_volunteer_message, get_user_from_slack_id + for user_ref in team_data.get("users", []): + user_data = user_ref.get().to_dict() + slack_user_id = user_data.get("user_id", "") + if slack_user_id: + send_volunteer_message( + admin_user=admin_user, + recipient_id=slack_user_id, + subject=f"Message from Opportunity Hack Team {team_data['name']}", + message=message + ) + + # If communication_history doesn't exist, create it and append the timestamp, sender, text if "communication_history" not in team_data: team_data["communication_history"] = [] diff --git a/common/log.py b/common/log.py index b072f63..cca2b5a 100644 --- a/common/log.py +++ b/common/log.py @@ -6,6 +6,12 @@ import sys from typing import Any, Dict, Optional, Union +try: + import colorlog + HAS_COLORLOG = True +except ImportError: + HAS_COLORLOG = False + # Configure default log level log_level = logging.INFO @@ -65,6 +71,14 @@ def format(self, record: logging.LogRecord) -> str: # Determine if we should use JSON logging based on environment use_json_logging = os.environ.get('USE_JSON_LOGGING', 'false').lower() == 'true' +# Determine if we should use colored logging (only in non-JSON mode) +# Default to True if colorlog is available and we're not using JSON logging +use_colored_logging = ( + HAS_COLORLOG and + not use_json_logging and + os.environ.get('USE_COLORED_LOGS', 'true').lower() == 'true' +) + # Root logger configuration def configure_root_logger(): """Configure the root logger with appropriate handlers""" @@ -82,9 +96,24 @@ def configure_root_logger(): # Apply appropriate formatter if use_json_logging: formatter = JsonFormatter() + elif use_colored_logging: + # Use colorlog for colored output with nice formatting + formatter = colorlog.ColoredFormatter( + '%(log_color)s%(asctime)s - %(name)s - %(levelname)s%(reset)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + log_colors={ + 'DEBUG': 'cyan', + 'INFO': 'green', + 'WARNING': 'yellow', + 'ERROR': 'red', + 'CRITICAL': 'red,bg_white', + }, + secondary_log_colors={}, + style='%' + ) else: formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - + console_handler.setFormatter(formatter) root_logger.addHandler(console_handler) diff --git a/requirements.txt b/requirements.txt index 0181c77..8610711 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,4 +43,5 @@ resend==2.3.0 readme-metrics[Flask]==3.1.0 redis==5.2.1 tiktoken==0.9.0 -numpy==1.26.3 \ No newline at end of file +numpy==1.26.3 +colorlog==6.7.0 \ No newline at end of file diff --git a/scripts/add_new_praise_columns.py b/scripts/add_new_praise_columns.py new file mode 100755 index 0000000..02304da --- /dev/null +++ b/scripts/add_new_praise_columns.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +""" +Backfill script to populate praise_receiver_ohack_id and praise_sender_ohack_id +in the praises collection. + +This script queries the praises collection, finds records missing these IDs, +and looks up the corresponding user records using their Slack IDs. +""" + +import sys +import os + +from dotenv import load_dotenv +load_dotenv() + +# Add parent directory to path to import from project modules +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from common.utils.firebase import get_db +import logging + +# Set up logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + +SLACK_PREFIX = "oauth2|slack|T1Q7936BH-" + + +def fetch_user_by_slack_id(db, slack_user_id, cache=None): + """ + Fetch a user from the users collection by their Slack user ID. + Uses an optional cache to avoid repeated database queries. + + Args: + db: Firestore database client + slack_user_id (str): Slack user ID (without prefix) + cache (dict, optional): Dictionary to cache user lookups + + Returns: + dict: User document with 'id' field, or None if not found + """ + # Add prefix if not present + if not slack_user_id.startswith(SLACK_PREFIX): + full_slack_id = f"{SLACK_PREFIX}{slack_user_id}" + else: + full_slack_id = slack_user_id + + # Check cache first + if cache is not None and full_slack_id in cache: + logger.debug(f"Cache hit for slack_id: {full_slack_id}") + return cache[full_slack_id] + + logger.debug(f"Cache miss - looking up user with slack_id: {full_slack_id}") + + # Query users collection + docs = db.collection('users').where("user_id", "==", full_slack_id).stream() + + for doc in docs: + user_dict = doc.to_dict() + user_dict['id'] = doc.id + logger.debug(f"Found user: {user_dict.get('name', 'Unknown')} (ID: {doc.id})") + + # Store in cache if provided + if cache is not None: + cache[full_slack_id] = user_dict + + return user_dict + + logger.warning(f"User not found for slack_id: {full_slack_id}") + + # Cache the negative result to avoid repeated lookups + if cache is not None: + cache[full_slack_id] = None + + return None + + +def backfill_praise_columns(): + """ + Main function to backfill praise_receiver_ohack_id and praise_sender_ohack_id + for all praises missing these fields. + + Optimizations: + - Uses in-memory cache to avoid duplicate user lookups + - Batches database writes (up to 500 operations per batch) + - Two-pass approach: collect data, then batch update + """ + logger.info("Starting praise column backfill script") + + # Get database connection + db = get_db() + + # Initialize user lookup cache + user_cache = {} + + # Query all praises + logger.info("Querying praises collection...") + praises_ref = db.collection('praises') + all_praises = list(praises_ref.stream()) + + logger.info(f"Found {len(all_praises)} total praises") + + # Counters for statistics + total_count = 0 + updated_count = 0 + skipped_count = 0 + error_count = 0 + cache_hit_count = 0 + cache_miss_count = 0 + + # Collect updates to batch + updates_to_apply = [] + + # First pass: Analyze all praises and prepare updates + logger.info("Pass 1: Analyzing praises and looking up users...") + for praise_doc in all_praises: + total_count += 1 + praise_data = praise_doc.to_dict() + praise_id = praise_doc.id + + if total_count % 50 == 0: + logger.info(f"Processed {total_count}/{len(all_praises)} praises...") + + # Check if both fields are already populated + has_receiver_id = 'praise_receiver_ohack_id' in praise_data and praise_data['praise_receiver_ohack_id'] + has_sender_id = 'praise_sender_ohack_id' in praise_data and praise_data['praise_sender_ohack_id'] + + if has_receiver_id and has_sender_id: + skipped_count += 1 + continue + + # Prepare update data + update_data = {} + + # Look up receiver if needed + if not has_receiver_id: + praise_receiver = praise_data.get('praise_receiver') + if praise_receiver: + receiver_user = fetch_user_by_slack_id(db, praise_receiver, cache=user_cache) + if receiver_user: + update_data['praise_receiver_ohack_id'] = receiver_user['id'] + else: + logger.warning(f" Praise {praise_id}: Could not find receiver user for slack_id: {praise_receiver}") + error_count += 1 + + # Look up sender if needed + if not has_sender_id: + praise_sender = praise_data.get('praise_sender') + if praise_sender: + sender_user = fetch_user_by_slack_id(db, praise_sender, cache=user_cache) + if sender_user: + update_data['praise_sender_ohack_id'] = sender_user['id'] + else: + logger.warning(f" Praise {praise_id}: Could not find sender user for slack_id: {praise_sender}") + error_count += 1 + + # Queue update if we have data to update + if update_data: + updates_to_apply.append({ + 'reference': praise_doc.reference, + 'data': update_data, + 'praise_id': praise_id + }) + + logger.info(f"Pass 1 complete: {len(updates_to_apply)} praises need updating") + logger.info(f"User cache contains {len(user_cache)} unique users") + + # Second pass: Apply updates in batches + if updates_to_apply: + logger.info("Pass 2: Applying updates in batches...") + batch_size = 500 # Firestore limit + num_batches = (len(updates_to_apply) + batch_size - 1) // batch_size + + for batch_num in range(num_batches): + start_idx = batch_num * batch_size + end_idx = min(start_idx + batch_size, len(updates_to_apply)) + batch_updates = updates_to_apply[start_idx:end_idx] + + logger.info(f"Processing batch {batch_num + 1}/{num_batches} ({len(batch_updates)} updates)...") + + # Create Firestore batch + batch = db.batch() + + for update in batch_updates: + batch.update(update['reference'], update['data']) + + # Commit the batch + try: + batch.commit() + updated_count += len(batch_updates) + logger.info(f" Successfully committed batch {batch_num + 1}") + except Exception as e: + logger.error(f" Error committing batch {batch_num + 1}: {str(e)}") + error_count += len(batch_updates) + + # Print summary + logger.info("=" * 80) + logger.info("Backfill complete!") + logger.info(f"Total praises processed: {total_count}") + logger.info(f"Successfully updated: {updated_count}") + logger.info(f"Skipped (already complete): {skipped_count}") + logger.info(f"Errors encountered: {error_count}") + logger.info(f"Unique users cached: {len(user_cache)}") + logger.info("=" * 80) + + +if __name__ == "__main__": + try: + backfill_praise_columns() + except Exception as e: + logger.error(f"Fatal error: {str(e)}", exc_info=True) + sys.exit(1) diff --git a/services/volunteers_service.py b/services/volunteers_service.py index e2b37ac..25e759d 100644 --- a/services/volunteers_service.py +++ b/services/volunteers_service.py @@ -1599,8 +1599,17 @@ def send_volunteer_message( volunteer_doc = db.collection('volunteers').document(volunteer_id).get() SLACK_USER_PREFIX = "oauth2|slack|T1Q7936BH-" + + # if recipient_id already has SLACK_USER_PREFIX, use it directly + if recipient_id and recipient_id.startswith(SLACK_USER_PREFIX): + pass + elif recipient_id: + recipient_id = f"{SLACK_USER_PREFIX}{recipient_id}" + else: + logger.warning("No recipient_id provided, Slack message may not be sent if slack_user_id is not found in volunteer record.") + # Search users for "user_id": "" - users_doc = db.collection('users').where('user_id', '==', f"{SLACK_USER_PREFIX}{recipient_id}").get() + users_doc = db.collection('users').where('user_id', '==', f"{recipient_id}").get() email = None slack_user_id = None