From 0442206f06b81babd2758241ead1a9a2968c2389 Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Mon, 2 Mar 2026 15:49:50 +0100 Subject: [PATCH 01/32] feat: add CI workflows for security, linting, and testing --- .github/workflows/ci-security-pipeline.yml | 104 +++++++++++++++++++++ .github/workflows/linter.yml | 30 ++++++ .github/workflows/tests.yml | 40 ++++++++ 3 files changed, 174 insertions(+) create mode 100644 .github/workflows/ci-security-pipeline.yml create mode 100644 .github/workflows/linter.yml create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/ci-security-pipeline.yml b/.github/workflows/ci-security-pipeline.yml new file mode 100644 index 0000000..27de289 --- /dev/null +++ b/.github/workflows/ci-security-pipeline.yml @@ -0,0 +1,104 @@ +name: CI Security Pipeline + +on: + push: + branches: [main] + pull_request: + branches: [main] + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12"] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: requirements.txt + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install pytest + + - name: Run tests + run: pytest + + lint: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Install Ruff + run: | + python -m pip install --upgrade pip + pip install ruff + + - name: Run Ruff + run: ruff check . + + bandit: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Install Bandit + run: | + python -m pip install --upgrade pip + pip install bandit + + - name: Run Bandit + run: bandit -r . -x tests -ll + + gitleaks: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository (full history) + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Run Gitleaks + uses: gitleaks/gitleaks-action@v2 + + semgrep: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Run Semgrep + uses: semgrep/semgrep-action@v1 + with: + config: p/security-audit diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml new file mode 100644 index 0000000..b8fb285 --- /dev/null +++ b/.github/workflows/linter.yml @@ -0,0 +1,30 @@ +name: Linter + +on: + push: + branches: + - "**" + pull_request: + +jobs: + black: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: "pip" + cache-dependency-path: "requirements.txt" + + - name: Install Black + run: | + python -m pip install --upgrade pip + pip install black + + - name: Run Black + run: black --check . diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..98e84c7 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,40 @@ +name: Tests + +on: + push: + branches: + - "**" + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: "pip" + cache-dependency-path: "requirements.txt" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install pytest pytest-mock pytest-cov + + - name: Run tests with coverage + run: pytest --cov=. --cov-report=term --cov-report=xml --cov-report=html + + - name: Upload coverage artifacts + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: | + coverage.xml + htmlcov/ + if-no-files-found: error From 56140e954569f8b7c57a8b6cff2b91372795ff49 Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Mon, 2 Mar 2026 15:53:30 +0100 Subject: [PATCH 02/32] fix broken file --- deprecated/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deprecated/test.py b/deprecated/test.py index 9c0b131..5ec84ee 100644 --- a/deprecated/test.py +++ b/deprecated/test.py @@ -103,4 +103,4 @@ def main(argv: List[str] | None = None) -> None: # 3. Resultat final print("\n===== RESULTAT =====") - print(f"Nom del projecte : {p + print(f"Nom del projecte : {project['name']}") \ No newline at end of file From 8d973ca6b7471df2e4700bd861103d36dd14dad6 Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Mon, 2 Mar 2026 15:55:00 +0100 Subject: [PATCH 03/32] refactor: pass black formatter --- app.py | 2 + config/credentials_loader.py | 8 +- config/logger_config.py | 1 + config/settings.py | 30 +- datasources/excel_handler.py | 59 ++-- datasources/github_handler.py | 112 +++--- datasources/requests/github_api_call.py | 17 +- datasources/requests/taiga_api_call.py | 58 ++-- datasources/taiga_handler.py | 325 ++++++++++-------- deprecated/test.py | 42 ++- routes/API_publisher/API_event_publisher.py | 16 +- routes/excel_routes.py | 32 +- routes/github_routes.py | 56 +-- routes/taiga_routes.py | 100 +++--- .../verify_signature_github.py | 14 +- .../verify_signature_taiga.py | 1 + tests/conftest.py | 58 ++-- tests/test_api_event_publisher.py | 2 + tests/test_app.py | 1 + tests/test_credentials_loader.py | 9 + tests/test_datetime_utils.py | 1 + tests/test_delete_webhooks_github.py | 9 +- tests/test_delete_webhooks_taiga.py | 9 +- tests/test_excel_handler.py | 11 +- tests/test_excel_routes.py | 21 +- tests/test_get_taiga_token.py | 1 + tests/test_github_api_call.py | 11 +- tests/test_github_handler.py | 142 +++++--- tests/test_github_recovery.py | 59 ++-- tests/test_github_routes.py | 71 ++-- tests/test_logger_config.py | 5 + tests/test_mongo_client.py | 3 + tests/test_settings.py | 10 +- tests/test_taiga_api_call.py | 44 ++- tests/test_taiga_auth.py | 2 + tests/test_taiga_handler.py | 58 +++- tests/test_taiga_recovery.py | 68 +++- tests/test_taiga_routes.py | 155 ++++++--- tests/test_verify_signature_github.py | 1 + tests/test_verify_signature_taiga.py | 1 + utils/datetime_utils.py | 2 +- utils/recovery/github_recovery.py | 282 ++++++++++----- utils/recovery/taiga_recovery.py | 322 ++++++++++------- utils/taiga_token/get_taiga_token.py | 11 +- utils/taiga_token/taiga_auth.py | 26 +- .../central_webhook_deletion.py | 9 +- .../delete_webhooks_github.py | 54 +-- .../webhook_deletion/delete_webhooks_taiga.py | 40 +-- 48 files changed, 1429 insertions(+), 942 deletions(-) diff --git a/app.py b/app.py index c204a75..f0e5328 100644 --- a/app.py +++ b/app.py @@ -8,6 +8,7 @@ setup_logging() logger = logging.getLogger(__name__) + def create_app(): app = Flask(__name__) @@ -18,6 +19,7 @@ def create_app(): logger.info("Flask created and Blueprints registered successfully.") return app + if __name__ == "__main__": app = create_app() app.run(debug=True, host="127.0.0.1", port=5000) diff --git a/config/credentials_loader.py b/config/credentials_loader.py index 43ae183..6a992c9 100644 --- a/config/credentials_loader.py +++ b/config/credentials_loader.py @@ -1,7 +1,7 @@ import json, os from typing import Optional -CONFIG_FILE = os.getenv("CREDENTIALS_FILE", - "config_files/credentials_config.json") + +CONFIG_FILE = os.getenv("CREDENTIALS_FILE", "config_files/credentials_config.json") def load(): @@ -16,9 +16,7 @@ def resolve(prj: str, field: str) -> Optional[str]: """ cfg = load() for course, props in cfg.items(): - if prj in props["teams"]: + if prj in props["teams"]: return props.get(field) raise KeyError(f"Project {prj!r} not found in {CONFIG_FILE}") - - diff --git a/config/logger_config.py b/config/logger_config.py index f21c47f..e12524e 100644 --- a/config/logger_config.py +++ b/config/logger_config.py @@ -4,6 +4,7 @@ _DEFAULT_FMT = "%(asctime)s [%(levelname)s] %(name)s: %(message)s" + def setup_logging() -> None: """ Configure the logger. diff --git a/config/settings.py b/config/settings.py index 33eb57e..ffc40ca 100644 --- a/config/settings.py +++ b/config/settings.py @@ -6,7 +6,8 @@ BASE_DIR = Path(__file__).resolve().parent.parent # Load environment variables from the .env file -load_dotenv(BASE_DIR / '.env') +load_dotenv(BASE_DIR / ".env") + def _require_env(name: str) -> str: value = os.getenv(name) @@ -17,18 +18,21 @@ def _require_env(name: str) -> str: ) return value -#Mongo database settings -MONGO_HOST = os.getenv("MONGO_HOST", "mongodb") -MONGO_PORT = os.getenv("MONGO_PORT", "27017") -MONGO_DB = os.getenv("MONGO_DB", "mongo") -MONGO_USER = os.getenv("MONGO_USER", "") -MONGO_PASS = os.getenv("MONGO_PASS", "") -MONGO_AUTHSRC = os.getenv("MONGO_AUTHSRC", MONGO_DB) + +# Mongo database settings +MONGO_HOST = os.getenv("MONGO_HOST", "mongodb") +MONGO_PORT = os.getenv("MONGO_PORT", "27017") +MONGO_DB = os.getenv("MONGO_DB", "mongo") +MONGO_USER = os.getenv("MONGO_USER", "") +MONGO_PASS = os.getenv("MONGO_PASS", "") +MONGO_AUTHSRC = os.getenv("MONGO_AUTHSRC", MONGO_DB) if MONGO_USER and MONGO_PASS: - MONGO_URI = (f"mongodb://{MONGO_USER}:{MONGO_PASS}" - f"@{MONGO_HOST}:{MONGO_PORT}/{MONGO_DB}" - f"?authSource={MONGO_AUTHSRC}") + MONGO_URI = ( + f"mongodb://{MONGO_USER}:{MONGO_PASS}" + f"@{MONGO_HOST}:{MONGO_PORT}/{MONGO_DB}" + f"?authSource={MONGO_AUTHSRC}" + ) else: MONGO_URI = f"mongodb://{MONGO_HOST}:{MONGO_PORT}/{MONGO_DB}" @@ -42,9 +46,9 @@ def _require_env(name: str) -> str: TAIGA_TOKEN = os.getenv("TAIGA_TOKEN", "") TAIGA_SIGNATURE_KEY = _require_env("TAIGA_SIGNATURE_KEY") TAIGA_USERNAME = _require_env("TAIGA_USERNAME") -TAIGA_PASSWORD= _require_env("TAIGA_PASSWORD") +TAIGA_PASSWORD = _require_env("TAIGA_PASSWORD") # Load the webhook URLs from the environment to enable the deletion of webhooks WEBHOOK_URL_GITHUB = os.getenv("WEBHOOK_URL_GITHUB", "") -WEBHOOK_URL_TAIGA = os.getenv("WEBHOOK_URL_TAIGA", "") \ No newline at end of file +WEBHOOK_URL_TAIGA = os.getenv("WEBHOOK_URL_TAIGA", "") diff --git a/datasources/excel_handler.py b/datasources/excel_handler.py index 903a17c..cb88b24 100644 --- a/datasources/excel_handler.py +++ b/datasources/excel_handler.py @@ -10,38 +10,37 @@ "Desenvolupament", "Gestió de projecte", "Documentació", - "Presentació" + "Presentació", ] + def parse_excel_event(raw_payload: dict, prj, quality_model) -> dict: - ''' + """ Function to parse the payload from the Excel webhook and return a dict with the data - ''' + """ # Receive the paraemters from the API query quality_model = quality_model - + # Get the values from the payload - ts = raw_payload.get("timestamp") - iteration= raw_payload.get("iteration") - date = raw_payload.get("date") + ts = raw_payload.get("timestamp") + iteration = raw_payload.get("iteration") + date = raw_payload.get("date") duration = raw_payload.get("duration") activity = raw_payload.get("activity") - comment = raw_payload.get("comment") - epic = raw_payload.get("epic") - members = raw_payload.get("members", []) - hours = raw_payload.get("memberHours", []) - config = raw_payload.get("configRange", []) - + comment = raw_payload.get("comment") + epic = raw_payload.get("epic") + members = raw_payload.get("members", []) + hours = raw_payload.get("memberHours", []) + config = raw_payload.get("configRange", []) - # Clean the members list, sometimes there are empty strings + # Clean the members list, sometimes there are empty strings members_clean = [m.strip() for m in members if isinstance(m, str) and m.strip()] # Pair the members with their hours, if there are more members than hours, we will ignore the extra members - hours_clean = hours[:len(members_clean)] + hours_clean = hours[: len(members_clean)] # Create a dictionary with the members and their hours members_dict = { - f"hours_{member}": hours_clean[idx] - for idx, member in enumerate(members_clean) + f"hours_{member}": hours_clean[idx] for idx, member in enumerate(members_clean) } # Map the activity types to the configRange values @@ -54,24 +53,20 @@ def parse_excel_event(raw_payload: dict, prj, quality_model) -> dict: # Sum the hours for each activity type total_hours = sum(activity_hours.values()) - - # Finally, return a dict containing the full structure return { - "timestamp": ts, - "team": prj, - "quality_model": quality_model, - "iteration": iteration, - "activity_date": date, - "duration_h": duration, - "activity_type": activity, - "comment": comment, - "epic": epic, - "members": members_clean, + "timestamp": ts, + "team": prj, + "quality_model": quality_model, + "iteration": iteration, + "activity_date": date, + "duration_h": duration, + "activity_type": activity, + "comment": comment, + "epic": epic, + "members": members_clean, **members_dict, **activity_hours, - "total_hours": total_hours + "total_hours": total_hours, } - - diff --git a/datasources/github_handler.py b/datasources/github_handler.py index 306996a..470568e 100644 --- a/datasources/github_handler.py +++ b/datasources/github_handler.py @@ -4,6 +4,7 @@ from config.credentials_loader import resolve from utils.datetime_utils import to_madrid_local + def parse_github_event(raw_payload: Dict, prj: str) -> Dict: """ Parse a GitHub event payload into a more detailed structure. @@ -21,12 +22,13 @@ def parse_github_event(raw_payload: Dict, prj: str) -> Dict: return {"event": event_type, "ignored": True} - def parse_github_push_event(raw_payload: Dict, prj: str) -> Dict: - ''' + """ Function to parse a GitHub push event payload. - ''' - event_type = "commit" # The event type is "push" but we will call it "commit" in our system + """ + event_type = ( + "commit" # The event type is "push" but we will call it "commit" in our system + ) # Retrieve the team name and repo name from the payload team_name = raw_payload.get("organization", {}).get("login", "UnknownTeam") @@ -39,9 +41,9 @@ def parse_github_push_event(raw_payload: Dict, prj: str) -> Dict: "login": sender.get("login", ""), "url": sender.get("url", ""), "type": sender.get("type", ""), - "site_admin": sender.get("site_admin", False) + "site_admin": sender.get("site_admin", False), } - + commits_info = [] # The push event typically has a "commits" array for c in raw_payload.get("commits", []): @@ -49,21 +51,20 @@ def parse_github_push_event(raw_payload: Dict, prj: str) -> Dict: commit_sha = c.get("id") commit_url = c.get("url", "") message = c.get("message", "") - #get timestamp of comit in the hour in spain - date= to_madrid_local(c.get("timestamp")) + # get timestamp of comit in the hour in spain + date = to_madrid_local(c.get("timestamp")) # Built author information author_login = c.get("author", {}).get("username", "") - author_name = c.get("author", {}).get("name", "") + author_name = c.get("author", {}).get("name", "") author_email = c.get("author", {}).get("email", "") # Compute message stats message_char_count = len(message) message_word_count = len(message.split()) - # Check if the commit message contains a task reference, it can be in english or catalan, ¿spanish¿ (e.g., "task #123") - pattern = r'(?i)\b(?:task|tasca)\b(?:\s*#?\s*(\d+))?' + pattern = r"(?i)\b(?:task|tasca)\b(?:\s*#?\s*(\d+))?" match = re.search(pattern, message) if match: task_is_written = True @@ -76,8 +77,8 @@ def parse_github_push_event(raw_payload: Dict, prj: str) -> Dict: verified = "false" verified_reason = "unsigned" - - commit_stats = fetch_commit_stats(repo_name, commit_sha, prj) + + commit_stats = fetch_commit_stats(repo_name, commit_sha, prj) # Build a final doc for this commit. commit_doc = { @@ -93,11 +94,11 @@ def parse_github_push_event(raw_payload: Dict, prj: str) -> Dict: "message": message, "message_char_count": message_char_count, "message_word_count": message_word_count, - "task_is_written": task_is_written, - "task_reference": task_reference, + "task_is_written": task_is_written, + "task_reference": task_reference, "verified": verified, "verified_reason": verified_reason, - "stats": commit_stats + "stats": commit_stats, } commits_info.append(commit_doc) @@ -108,20 +109,18 @@ def parse_github_push_event(raw_payload: Dict, prj: str) -> Dict: "repo_name": repo_name, "team_name": team_name, "sender_info": sender_info, - "commits": commits_info + "commits": commits_info, } - - def parse_github_issue_event(raw_payload: Dict, prj: str) -> Dict: - ''' + """ Function to parse a GitHub issue event payload. - ''' + """ # Retrieve the event information from the payload - action = raw_payload.get("action", "unknown-action")# e.g. "issue_opened" - event_type = "issue" - + action = raw_payload.get("action", "unknown-action") # e.g. "issue_opened" + event_type = "issue" + # Retrieve the team name and repo name from the payload team_name = raw_payload.get("organization", {}).get("login", "UnknownTeam") repo_name = raw_payload["repository"].get("full_name", "unknown-repo") @@ -133,16 +132,16 @@ def parse_github_issue_event(raw_payload: Dict, prj: str) -> Dict: "login": sender.get("login", ""), "url": sender.get("url", ""), "type": sender.get("type", ""), - "site_admin": sender.get("site_admin", False) + "site_admin": sender.get("site_admin", False), } # The "issue" object is typically raw_payload["issue"] issue_data = raw_payload.get("issue", {}) issue_number = issue_data.get("number", 0) - issue_title = issue_data.get("title", "") - issue_state = issue_data.get("state", "") - issue_body = issue_data.get("body", "") - issue_user = issue_data.get("user", {}) + issue_title = issue_data.get("title", "") + issue_state = issue_data.get("state", "") + issue_body = issue_data.get("body", "") + issue_user = issue_data.get("user", {}) issue_user_login = issue_user.get("login", "") issue_user_id = issue_user.get("id", "") @@ -152,37 +151,31 @@ def parse_github_issue_event(raw_payload: Dict, prj: str) -> Dict: "title": issue_title, "state": issue_state, "body": issue_body, - "user": { - "login": issue_user_login, - "id": issue_user_id - } + "user": {"login": issue_user_login, "id": issue_user_id}, } # Finally, return a dict containing the full structure return { - "event": event_type, - "action": action, + "event": event_type, + "action": action, "repo_name": repo_name, "team_name": team_name, "sender_info": sender_info, - "issue": issue_obj + "issue": issue_obj, } - def parse_github_pullrequest_event(raw_payload: Dict, prj: str) -> Dict: - ''' + """ Function to parse a GitHub pull request event payload. - ''' - + """ + action = raw_payload.get("action") - - if action != "closed": # we only want to process closed pull requests - return {"event": "pull_request", "ignored": True} - - event_type = "pull_request" # The event type is "pull request" but we will call it "commit" in our system + if action != "closed": # we only want to process closed pull requests + return {"event": "pull_request", "ignored": True} + event_type = "pull_request" # The event type is "pull request" but we will call it "commit" in our system # The 'sender' object is at the top level sender = raw_payload.get("sender", {}) @@ -191,31 +184,29 @@ def parse_github_pullrequest_event(raw_payload: Dict, prj: str) -> Dict: "login": sender.get("login", ""), "url": sender.get("url", ""), "type": sender.get("type", ""), - "site_admin": sender.get("site_admin", False) + "site_admin": sender.get("site_admin", False), } - + # Get the information about the pull request pr_info = raw_payload.get("pull_request", {}) - + pr_number = pr_info.get("number", 0) - pr_title = pr_info.get("title", "") + pr_title = pr_info.get("title", "") pr_created_at = to_madrid_local(pr_info.get("created_at", "")) - pr_closed_at = to_madrid_local(pr_info.get("closed_at", "")) + pr_closed_at = to_madrid_local(pr_info.get("closed_at", "")) pr_merged_at = pr_info.get("merged", False) merged_by = pr_info.get("merged_by", {}).get("login", "") - - - pr_assignee = (pr_info.get("assignee") or {}).get("login"), + + pr_assignee = ((pr_info.get("assignee") or {}).get("login"),) pr_reviewers = [r["login"] for r in pr_info.get("requested_reviewers", [])] - - + team_name = raw_payload.get("organization", {}).get("login", "UnknownTeam") repo_name = raw_payload["repository"].get("full_name", "unknown-repo") - - # Build a final doc for this commit. + + # Build a final doc for this commit. pr_doc = { "event": event_type, - "action": action, # always "closed" + "action": action, # always "closed" "pr_number": pr_number, "title": pr_title, "created_at": pr_created_at, @@ -227,8 +218,7 @@ def parse_github_pullrequest_event(raw_payload: Dict, prj: str) -> Dict: "repo_name": repo_name, "team_name": team_name, "sender_info": sender_info, - } - + } # Finally, return a dict containing the full structure - return pr_doc \ No newline at end of file + return pr_doc diff --git a/datasources/requests/github_api_call.py b/datasources/requests/github_api_call.py index 1827cc6..d26a145 100644 --- a/datasources/requests/github_api_call.py +++ b/datasources/requests/github_api_call.py @@ -8,19 +8,20 @@ logger = logging.getLogger(__name__) -def fetch_commit_stats(repo_full_name: str, commit_sha: str, prj: str) -> Dict[str, int]: +def fetch_commit_stats( + repo_full_name: str, commit_sha: str, prj: str +) -> Dict[str, int]: """ Gets 'additions', 'deletions' y 'total' of a commit using GitHub's REST API v3. """ - + token = resolve(prj, "github_token") - + headers = { "Accept": "application/vnd.github.v3+json", - "Authorization": f"token {token}" + "Authorization": f"token {token}", } - url = f"{GITHUB_API_URL}/repos/{repo_full_name}/commits/{commit_sha}" try: @@ -30,8 +31,10 @@ def fetch_commit_stats(repo_full_name: str, commit_sha: str, prj: str) -> Dict[s return { "total": stats.get("total", 0), "additions": stats.get("additions", 0), - "deletions": stats.get("deletions", 0) + "deletions": stats.get("deletions", 0), } except Exception as exc: - logger.error("Error fetching commit stats for %s/%s: %s", repo_full_name, commit_sha, exc) + logger.error( + "Error fetching commit stats for %s/%s: %s", repo_full_name, commit_sha, exc + ) return {"total": 0, "additions": 0, "deletions": 0} diff --git a/datasources/requests/taiga_api_call.py b/datasources/requests/taiga_api_call.py index 1d9f603..31ed12c 100644 --- a/datasources/requests/taiga_api_call.py +++ b/datasources/requests/taiga_api_call.py @@ -8,26 +8,34 @@ logger = logging.getLogger(__name__) -_CACHE = {} # key = (project_id, milestone_id) -> (timestamp, stats) -TTL = timedelta(minutes=1) # Cache time-to-live, set to 5 minutes. Means that if the same request is made within 5 minutes, it will return the cached result instead of making a new API call. +_CACHE = {} # key = (project_id, milestone_id) -> (timestamp, stats) +TTL = timedelta( + minutes=1 +) # Cache time-to-live, set to 5 minutes. Means that if the same request is made within 5 minutes, it will return the cached result instead of making a new API call. + def milestone_stats(project_id: str, milestone_id: str, prj: str): - ''' - Fetches the statistics of a milestone in a Taiga project. + """ + Fetches the statistics of a milestone in a Taiga project. Uses caching to avoid frequent API calls to get the taiga token. It refreshes the cache every 5 minutes. - ''' + """ if not project_id or not milestone_id: return {} key = (project_id, milestone_id) now = datetime.utcnow() - if key in _CACHE and now - _CACHE[key][0]< TTL: + if key in _CACHE and now - _CACHE[key][0] < TTL: return _CACHE[key][1] user = resolve(prj, "taiga_user") - psw = resolve(prj, "taiga_password") - logger.debug("Resolving Taiga credentials for project %s: user=%s, password=%s", prj, "****" if user else None, "****" if psw else None) + psw = resolve(prj, "taiga_password") + logger.debug( + "Resolving Taiga credentials for project %s: user=%s, password=%s", + prj, + "****" if user else None, + "****" if psw else None, + ) if user and psw: token = get_taiga_token(user, psw) headers = {"Authorization": f"Bearer {token}"} @@ -35,35 +43,37 @@ def milestone_stats(project_id: str, milestone_id: str, prj: str): else: headers = {} logger.info("Using Taiga tunnel without authentication for project: %s", prj) - + url = f"{TAIGA_API_URL}/milestones/{milestone_id}/stats" logger.debug("Fetching Taiga milestone stats from URL: %s", url) - r = requests.get(url, params={"project": project_id}, headers=headers, timeout=(1, 5)) - + r = requests.get( + url, params={"project": project_id}, headers=headers, timeout=(1, 5) + ) + try: r.raise_for_status() except requests.exceptions.HTTPError as e: print(f"Warning: Failed to fetch milestone stats (status {r.status_code}): {e}") # Return empty stats if we can't access the milestone stats = { - "milestone_total_points" : 0, - "milestone_closed_points" : 0, - "milestone_total_userstories" : 0, + "milestone_total_points": 0, + "milestone_closed_points": 0, + "milestone_total_userstories": 0, "milestone_completed_userstories": 0, - "milestone_total_tasks" : 0, - "milestone_completed_tasks" : 0, + "milestone_total_tasks": 0, + "milestone_completed_tasks": 0, } _CACHE[key] = (now, stats) return stats - - js = r.json() + + js = r.json() stats = { - "milestone_total_points" : sum(js.get("total_points", {}).values()), - "milestone_closed_points" : sum(js.get("completed_points", 0)), - "milestone_total_userstories" : js.get("total_userstories", 0), + "milestone_total_points": sum(js.get("total_points", {}).values()), + "milestone_closed_points": sum(js.get("completed_points", 0)), + "milestone_total_userstories": js.get("total_userstories", 0), "milestone_completed_userstories": js.get("completed_userstories", 0), - "milestone_total_tasks" : js.get("total_tasks", 0), - "milestone_completed_tasks" : js.get("completed_tasks", 0), + "milestone_total_tasks": js.get("total_tasks", 0), + "milestone_completed_tasks": js.get("completed_tasks", 0), } _CACHE[key] = (now, stats) - return stats \ No newline at end of file + return stats diff --git a/datasources/taiga_handler.py b/datasources/taiga_handler.py index 43d3dc3..62858e0 100644 --- a/datasources/taiga_handler.py +++ b/datasources/taiga_handler.py @@ -1,7 +1,7 @@ from typing import Dict import re -from datasources.requests.taiga_api_call import milestone_stats +from datasources.requests.taiga_api_call import milestone_stats from utils.datetime_utils import to_madrid_local @@ -12,7 +12,7 @@ def parse_taiga_event(raw_payload: Dict, prj: str) -> Dict: We can handle "issues", "epics", "tasks" and "userstories" events. """ event_type = raw_payload.get("type") - if event_type == "issue": + if event_type == "issue": return parse_taiga_issue_event(raw_payload, prj) elif event_type == "epic": return parse_taiga_epic_event(raw_payload, prj) @@ -23,42 +23,43 @@ def parse_taiga_event(raw_payload: Dict, prj: str) -> Dict: elif event_type == "relateduserstory": return parse_taiga_related_userstory_event(raw_payload, prj) else: - return { - "event": event_type, - "error": "Unsupported event type" - } + return {"event": event_type, "error": "Unsupported event type"} - - -def parse_taiga_issue_event(raw_payload: Dict, prj: str) -> Dict: - ''' +def parse_taiga_issue_event(raw_payload: Dict, prj: str) -> Dict: + """ Function to parse a taiga issue event payload. - ''' + """ # Extract the relevant fields from the raw payload - project_id= raw_payload.get("data",{}).get("project",{}).get("id", "") - issue_id=raw_payload.get("data",{}).get("id", "") - team_name = raw_payload.get("data",{}).get("project", {}).get("name", "") - event_type= raw_payload.get("type","") - action_type= raw_payload.get("action","") - subject = raw_payload.get("data",{}).get("subject", "") - due_date = to_madrid_local(raw_payload.get("data",{}).get("due_date", "")) - description = raw_payload.get("data", {}).get("description", "") + project_id = raw_payload.get("data", {}).get("project", {}).get("id", "") + issue_id = raw_payload.get("data", {}).get("id", "") + team_name = raw_payload.get("data", {}).get("project", {}).get("name", "") + event_type = raw_payload.get("type", "") + action_type = raw_payload.get("action", "") + subject = raw_payload.get("data", {}).get("subject", "") + due_date = to_madrid_local(raw_payload.get("data", {}).get("due_date", "")) + description = raw_payload.get("data", {}).get("description", "") severity = raw_payload.get("data", {}).get("severity", {}).get("name", "") status = raw_payload.get("data", {}).get("status", {}).get("name", "") priority = raw_payload.get("data", {}).get("priority", {}).get("name", "") - type = raw_payload.get("data", {}).get("type", {}).get("name", "") + type = raw_payload.get("data", {}).get("type", {}).get("name", "") is_closed = raw_payload.get("is_closed", False) - modified_date = to_madrid_local(raw_payload.get("data", {}).get("modified_date", "")) + modified_date = to_madrid_local( + raw_payload.get("data", {}).get("modified_date", "") + ) created_date = to_madrid_local(raw_payload.get("data", {}).get("created_date", "")) - finished_date = to_madrid_local(raw_payload.get("data", {}).get("finished_date", "")) + finished_date = to_madrid_local( + raw_payload.get("data", {}).get("finished_date", "") + ) assigned_by = raw_payload.get("by", {}).get("username", "") - #There are cases where the assigned_to field is empty, and if we request it aniways it will throw an error, so we need to check if it exists + # There are cases where the assigned_to field is empty, and if we request it aniways it will throw an error, so we need to check if it exists if raw_payload.get("data", {}).get("assigned_to", {}) != None: - assigned_to = raw_payload.get("data", {}).get("assigned_to", {}).get("username", "") + assigned_to = ( + raw_payload.get("data", {}).get("assigned_to", {}).get("username", "") + ) else: assigned_to = None - + # Create a dictionary with all the attributes of the issue doc = { "project_id": project_id, @@ -78,32 +79,32 @@ def parse_taiga_issue_event(raw_payload: Dict, prj: str) -> Dict: "created_date": created_date, "finished_date": finished_date, "assigned_by": assigned_by, - "assigned_to": assigned_to + "assigned_to": assigned_to, } # Return the parsed issue data as a dictionary return doc - - -def parse_taiga_epic_event(raw_payload: Dict, prj: str) -> Dict: - ''' +def parse_taiga_epic_event(raw_payload: Dict, prj: str) -> Dict: + """ Function to parse a taiga epic event payload. - ''' + """ # Extract the relevant fields from the raw payload - epic_id= raw_payload.get("data",{}).get("id", "") - team_name = raw_payload.get("data",{}).get("project", {}).get("name", "") - event_type= raw_payload.get("type","") - action_type= raw_payload.get("action","") - subject = raw_payload.get("data",{}).get("subject", "") + epic_id = raw_payload.get("data", {}).get("id", "") + team_name = raw_payload.get("data", {}).get("project", {}).get("name", "") + event_type = raw_payload.get("type", "") + action_type = raw_payload.get("action", "") + subject = raw_payload.get("data", {}).get("subject", "") status = raw_payload.get("data", {}).get("status", {}).get("name", "") is_closed = raw_payload.get("is_closed", False) - modified_date = to_madrid_local(raw_payload.get("data", {}).get("modified_date", "")) + modified_date = to_madrid_local( + raw_payload.get("data", {}).get("modified_date", "") + ) created_date = to_madrid_local(raw_payload.get("data", {}).get("created_date", "")) assigned_by = raw_payload.get("by", {}).get("username", "") - #We are going to use this project_id to delete the webhooks with the TAIGA API - project_id= raw_payload.get("data",{}).get("project",{}).get("id", "") - + # We are going to use this project_id to delete the webhooks with the TAIGA API + project_id = raw_payload.get("data", {}).get("project", {}).get("id", "") + # Create a dictionary with all the attributes of the epic doc = { "epic_id": epic_id, @@ -116,58 +117,76 @@ def parse_taiga_epic_event(raw_payload: Dict, prj: str) -> Dict: "status": status, "modified_date": modified_date, "created_date": created_date, - "project_id": project_id - + "project_id": project_id, } # Return the parsed epic data as a dictionary return doc - - -def parse_taiga_task_event(raw_payload: Dict, prj: str) -> Dict: - ''' +def parse_taiga_task_event(raw_payload: Dict, prj: str) -> Dict: + """ Function to parse a taiga task event payload. - ''' + """ # Extract the relevant fields from the raw payload - project_id= raw_payload.get("data",{}).get("project",{}).get("id", "") - team_name = raw_payload.get("data",{}).get("project", {}).get("name", "") - event_type= raw_payload.get("type","") - action_type= raw_payload.get("action","") - task_id= raw_payload.get("data",{}).get("id", "") - subject = raw_payload.get("data",{}).get("subject", "") - userstory_id= raw_payload.get("data",{}).get("user_story",{}).get("id", "") - userstory_is_closed= raw_payload.get("data",{}).get("user_story",{}).get("is_closed", "") + project_id = raw_payload.get("data", {}).get("project", {}).get("id", "") + team_name = raw_payload.get("data", {}).get("project", {}).get("name", "") + event_type = raw_payload.get("type", "") + action_type = raw_payload.get("action", "") + task_id = raw_payload.get("data", {}).get("id", "") + subject = raw_payload.get("data", {}).get("subject", "") + userstory_id = raw_payload.get("data", {}).get("user_story", {}).get("id", "") + userstory_is_closed = ( + raw_payload.get("data", {}).get("user_story", {}).get("is_closed", "") + ) is_closed = raw_payload.get("data", {}).get("status", {}).get("is_closed", "") status = raw_payload.get("data", {}).get("status", {}).get("name", "") created_date = to_madrid_local(raw_payload.get("data", {}).get("created_date", "")) - modified_date = to_madrid_local(raw_payload.get("data", {}).get("modified_date", "")) - finished_date = to_madrid_local(raw_payload.get("data", {}).get("finished_date", "")) - reference=raw_payload.get("data",{}).get("ref", "") - milestone_id=raw_payload.get("data",{}).get("milestone",{}).get("id", "") - milestone_name=raw_payload.get("data",{}).get("milestone",{}).get("name", "") - milestone_closed=raw_payload.get("data",{}).get("milestone",{}).get("closed", "") - milestone_created_date=raw_payload.get("data",{}).get("milestone",{}).get("created_date", "") - milestone_created_date = to_madrid_local(milestone_created_date) if milestone_created_date else "" - milestone_modified_date=raw_payload.get("data",{}).get("milestone",{}).get("modified_date", "") - milestone_modified_date = to_madrid_local(milestone_modified_date) if milestone_modified_date else "" - estimated_start=to_madrid_local(raw_payload.get("data",{}).get("milestone",{}).get("estimated_start", "")) - estimated_finish=to_madrid_local(raw_payload.get("data",{}).get("milestone",{}).get("estimated_finish", "")) + modified_date = to_madrid_local( + raw_payload.get("data", {}).get("modified_date", "") + ) + finished_date = to_madrid_local( + raw_payload.get("data", {}).get("finished_date", "") + ) + reference = raw_payload.get("data", {}).get("ref", "") + milestone_id = raw_payload.get("data", {}).get("milestone", {}).get("id", "") + milestone_name = raw_payload.get("data", {}).get("milestone", {}).get("name", "") + milestone_closed = ( + raw_payload.get("data", {}).get("milestone", {}).get("closed", "") + ) + milestone_created_date = ( + raw_payload.get("data", {}).get("milestone", {}).get("created_date", "") + ) + milestone_created_date = ( + to_madrid_local(milestone_created_date) if milestone_created_date else "" + ) + milestone_modified_date = ( + raw_payload.get("data", {}).get("milestone", {}).get("modified_date", "") + ) + milestone_modified_date = ( + to_madrid_local(milestone_modified_date) if milestone_modified_date else "" + ) + estimated_start = to_madrid_local( + raw_payload.get("data", {}).get("milestone", {}).get("estimated_start", "") + ) + estimated_finish = to_madrid_local( + raw_payload.get("data", {}).get("milestone", {}).get("estimated_finish", "") + ) assigned_by = raw_payload.get("by", {}).get("username", "") - - + milestone_data = milestone_stats(project_id, milestone_id, prj) - #If someone defines a new metric, if it isnt listed in the handler, we wont get it. To solve we can get all the custom attributes as an object and store it in mongo + # If someone defines a new metric, if it isnt listed in the handler, we wont get it. To solve we can get all the custom attributes as an object and store it in mongo custom_attributes = raw_payload.get("data", {}).get("custom_attributes_values", {}) if custom_attributes is None: custom_attributes = {} - - #There are cases where the assigned_to field is empty, and if we request it aniways it will throw an error, so we need to check if it exists + + # There are cases where the assigned_to field is empty, and if we request it aniways it will throw an error, so we need to check if it exists if raw_payload.get("data", {}).get("assigned_to", {}) != None: - assigned_to = raw_payload.get("data", {}).get("assigned_to", {}).get("username", "") + assigned_to = ( + raw_payload.get("data", {}).get("assigned_to", {}).get("username", "") + ) else: assigned_to = None - + # Create a dictionary with all the attributes of the task doc = { "project_id": project_id, @@ -193,83 +212,94 @@ def parse_taiga_task_event(raw_payload: Dict, prj: str) -> Dict: "milestone_modified_date": milestone_modified_date, "estimated_start": estimated_start, "estimated_finish": estimated_finish, - - #We can get all the custom attributes like an object, but in mongo they will have the name defined in taiga. - "custom_attributes": custom_attributes, + # We can get all the custom attributes like an object, but in mongo they will have the name defined in taiga. + "custom_attributes": custom_attributes, } doc.update(milestone_data) # Return the parsed task data as a dictionary return doc - - - - - -#Most fields dont appear when creating the user story from zero, they appear once we link it to an epic -def parse_taiga_userstory_event(raw_payload: Dict, prj: str) -> Dict: - ''' +# Most fields dont appear when creating the user story from zero, they appear once we link it to an epic +def parse_taiga_userstory_event(raw_payload: Dict, prj: str) -> Dict: + """ Function to parse a taiga userstory event payload. - ''' + """ # Extract the relevant fields from the raw payload - project_id= raw_payload.get("data",{}).get("project",{}).get("id", "") - userstory_id= raw_payload.get("data",{}).get("id", "") - team_name = raw_payload.get("data",{}).get("project", {}).get("name", "") - event_type= raw_payload.get("type","") - action_type= raw_payload.get("action","") - subject = raw_payload.get("data",{}).get("subject", "") + project_id = raw_payload.get("data", {}).get("project", {}).get("id", "") + userstory_id = raw_payload.get("data", {}).get("id", "") + team_name = raw_payload.get("data", {}).get("project", {}).get("name", "") + event_type = raw_payload.get("type", "") + action_type = raw_payload.get("action", "") + subject = raw_payload.get("data", {}).get("subject", "") status = raw_payload.get("data", {}).get("status", {}).get("name", "") is_closed = raw_payload.get("is_closed", False) - modified_date = to_madrid_local(raw_payload.get("data", {}).get("modified_date", "")) + modified_date = to_madrid_local( + raw_payload.get("data", {}).get("modified_date", "") + ) created_date = to_madrid_local(raw_payload.get("data", {}).get("created_date", "")) assigned_by = raw_payload.get("by", {}).get("username", "") # Extract all the custom attributes from the payload, if they exist - custom_attributes = raw_payload.get("data", {}).get("custom_attributes_values", {}) + custom_attributes = raw_payload.get("data", {}).get("custom_attributes_values", {}) if custom_attributes is None: custom_attributes = {} - - description= raw_payload.get("data", {}).get("description", "") - #If the pattern "AS - A - I WANT - SO THAT" is used in the description, the vañue of pattern will be True, if not, it will be False + + description = raw_payload.get("data", {}).get("description", "") + # If the pattern "AS - A - I WANT - SO THAT" is used in the description, the vañue of pattern will be True, if not, it will be False pattern = r"as\s+(.*?)\s+i want\s+(.*?)\s+so that\s+(.*)" match = re.search(pattern, description, re.IGNORECASE) if match: pattern_in_description = True else: pattern_in_description = False - + # If the userstory has a milestone associated while created, we will get the values, if not, we will set them to None - if raw_payload.get("data",{}).get("milestone",{}) != None: - - milestone_id= raw_payload.get("data",{}).get("milestone",{}).get("id", "") - milestone_name= raw_payload.get("data",{}).get("milestone",{}).get("name", "") - milestone_closed= raw_payload.get("data",{}).get("milestone",{}).get("closed", "") - milestone_created_date= to_madrid_local(raw_payload.get("data",{}).get("milestone",{}).get("created_date", "")) - milestone_modified_date= to_madrid_local(raw_payload.get("data",{}).get("milestone",{}).get("modified_date", "")) - milestone_modified_date = to_madrid_local(milestone_modified_date) if milestone_modified_date else "" - estimated_start= to_madrid_local(raw_payload.get("data",{}).get("milestone",{}).get("estimated_start", "")) - estimated_finish= to_madrid_local(raw_payload.get("data",{}).get("milestone",{}).get("estimated_finish", "")) - - milestone_data= milestone_stats(project_id, milestone_id, prj) - - - + if raw_payload.get("data", {}).get("milestone", {}) != None: + + milestone_id = raw_payload.get("data", {}).get("milestone", {}).get("id", "") + milestone_name = ( + raw_payload.get("data", {}).get("milestone", {}).get("name", "") + ) + milestone_closed = ( + raw_payload.get("data", {}).get("milestone", {}).get("closed", "") + ) + milestone_created_date = to_madrid_local( + raw_payload.get("data", {}).get("milestone", {}).get("created_date", "") + ) + milestone_modified_date = to_madrid_local( + raw_payload.get("data", {}).get("milestone", {}).get("modified_date", "") + ) + milestone_modified_date = ( + to_madrid_local(milestone_modified_date) if milestone_modified_date else "" + ) + estimated_start = to_madrid_local( + raw_payload.get("data", {}).get("milestone", {}).get("estimated_start", "") + ) + estimated_finish = to_madrid_local( + raw_payload.get("data", {}).get("milestone", {}).get("estimated_finish", "") + ) + + milestone_data = milestone_stats(project_id, milestone_id, prj) + else: - milestone_id= "" - milestone_name= "" - milestone_closed= "" - milestone_created_date= "" - milestone_modified_date= "" - estimated_start= "" - estimated_finish= "" + milestone_id = "" + milestone_name = "" + milestone_closed = "" + milestone_created_date = "" + milestone_modified_date = "" + estimated_start = "" + estimated_finish = "" milestone_data = {} - - priority = raw_payload.get("data", {}).get("custom_attributes_values", {}).get("Priority", "") - points_list = raw_payload.get("data",{}).get("points", []) + priority = ( + raw_payload.get("data", {}) + .get("custom_attributes_values", {}) + .get("Priority", "") + ) + points_list = raw_payload.get("data", {}).get("points", []) sum_points = sum(p.get("value") or 0 for p in points_list) - # Create a dictionary with all the attributes of the user story + # Create a dictionary with all the attributes of the user story doc = { "project_id": project_id, "team_name": team_name, @@ -280,7 +310,7 @@ def parse_taiga_userstory_event(raw_payload: Dict, prj: str) -> Dict: "is_closed": is_closed, "status": status, "created_date": created_date, - "modified_date": modified_date, + "modified_date": modified_date, "total_points": sum_points, "assigned_by": assigned_by, "milestone_id": milestone_id, @@ -290,43 +320,45 @@ def parse_taiga_userstory_event(raw_payload: Dict, prj: str) -> Dict: "milestone_modified_date": milestone_modified_date, "estimated_start": estimated_start, "estimated_finish": estimated_finish, - #We can get all the custom attributes like an object, but in mongo they will have the name defined in taiga. - "custom_attributes": custom_attributes, - #"acceptance_criteria": acceptance_criteria, #TRUE IF THE USER STORY HAS ACCEPTANCE CRITERIA - "pattern": pattern_in_description, - "priority": priority + # We can get all the custom attributes like an object, but in mongo they will have the name defined in taiga. + "custom_attributes": custom_attributes, + # "acceptance_criteria": acceptance_criteria, #TRUE IF THE USER STORY HAS ACCEPTANCE CRITERIA + "pattern": pattern_in_description, + "priority": priority, } - + doc.update(milestone_data) # Return the parsed user story data as a dictionary return doc - - -def parse_taiga_related_userstory_event(raw_payload: Dict, prj: str) -> Dict: - ''' +def parse_taiga_related_userstory_event(raw_payload: Dict, prj: str) -> Dict: + """ Function to parse a taiga related userstory event payload. This related userstory event is triggered when a user story is linked to an epic. - ''' + """ # Extract the relevant fields from the raw payload - userstory_id= raw_payload.get("data",{}).get("user_story",{}).get("id", "") - team_name = raw_payload.get("data",{}).get("epic",{}).get("project", {}).get("name", "") - event_type= raw_payload.get("type","") - finished_date = to_madrid_local(raw_payload.get("data",{}).get("finished_date", "")) - assigned_to = raw_payload.get("data",{}).get("assigned_to",{}).get("username", "") - epic_id= raw_payload.get("data",{}).get("epic",{}).get("id", "") - epic_name= raw_payload.get("data",{}).get("epic",{}).get("subject","") - reference= raw_payload.get("data",{}).get("epic",{}).get("ref", "") + userstory_id = raw_payload.get("data", {}).get("user_story", {}).get("id", "") + team_name = ( + raw_payload.get("data", {}).get("epic", {}).get("project", {}).get("name", "") + ) + event_type = raw_payload.get("type", "") + finished_date = to_madrid_local( + raw_payload.get("data", {}).get("finished_date", "") + ) + assigned_to = raw_payload.get("data", {}).get("assigned_to", {}).get("username", "") + epic_id = raw_payload.get("data", {}).get("epic", {}).get("id", "") + epic_name = raw_payload.get("data", {}).get("epic", {}).get("subject", "") + reference = raw_payload.get("data", {}).get("epic", {}).get("ref", "") assigned_by = raw_payload.get("by", {}).get("username", "") # Create a dictionary with all the attributes of the user story doc = { - "id": userstory_id, + "id": userstory_id, "team_name": team_name, "event_type": event_type, "epic_id": epic_id, - "epic_name": epic_name, + "epic_name": epic_name, "reference": reference, "finished_date": finished_date, "assigned_to": assigned_to, @@ -334,4 +366,3 @@ def parse_taiga_related_userstory_event(raw_payload: Dict, prj: str) -> Dict: } # Return the parsed user story data as a dictionary return doc - diff --git a/deprecated/test.py b/deprecated/test.py index 5ec84ee..67d9348 100644 --- a/deprecated/test.py +++ b/deprecated/test.py @@ -30,14 +30,12 @@ API_BASE = "https://api.taiga.io/api/v1" - - - def get_by_slug(slug: str, token: str) -> Dict: """Retorna el JSON del projecte pel seu slug (una sola petició).""" h = {"Authorization": f"Bearer {token}"} - r = requests.get(f"{API_BASE}/projects/by_slug", headers=h, - params={"slug": slug}, timeout=10) + r = requests.get( + f"{API_BASE}/projects/by_slug", headers=h, params={"slug": slug}, timeout=10 + ) if r.status_code == 404: sys.exit(f"Slug «{slug}» no existeix o no és visible per aquest usuari.") r.raise_for_status() @@ -48,10 +46,11 @@ def list_user_projects(token: str) -> List[Dict]: """Llista tots els projectes on l’usuari autenticat és membre.""" h = { "Authorization": f"Bearer {token}", - "x-disable-pagination": "True", # rebem tot en una sola resposta + "x-disable-pagination": "True", # rebem tot en una sola resposta } - r = requests.get(f"{API_BASE}/projects", headers=h, - params={"member": "me"}, timeout=10) + r = requests.get( + f"{API_BASE}/projects", headers=h, params={"member": "me"}, timeout=10 + ) r.raise_for_status() return r.json() @@ -75,12 +74,20 @@ def main(argv: List[str] | None = None) -> None: description="Fetch Taiga project id by slug or by display name." ) ap.add_argument("--slug", help="Slug exacte del projecte (mètode preferit).") - ap.add_argument("--name", - help="Nom visible del projecte (case-insensitive). S'usa si --slug no és present.") - ap.add_argument("--user", default=os.getenv("TAIGA_USERNAME"), - help="Usuari Taiga (o variable d’entorn TAIGA_USERNAME).") - ap.add_argument("--password", default=os.getenv("TAIGA_PASSWORD"), - help="Contrasenya Taiga (o variable d’entorn TAIGA_PASSWORD).") + ap.add_argument( + "--name", + help="Nom visible del projecte (case-insensitive). S'usa si --slug no és present.", + ) + ap.add_argument( + "--user", + default=os.getenv("TAIGA_USERNAME"), + help="Usuari Taiga (o variable d’entorn TAIGA_USERNAME).", + ) + ap.add_argument( + "--password", + default=os.getenv("TAIGA_PASSWORD"), + help="Contrasenya Taiga (o variable d’entorn TAIGA_PASSWORD).", + ) args = ap.parse_args(argv) # Validacions bàsiques @@ -88,8 +95,9 @@ def main(argv: List[str] | None = None) -> None: ap.error("Cal indicar --slug o --name") if not (args.user and args.password): - ap.error("Credencials requerides: --user/--password o variables env TAIGA_USERNAME/TAIGA_PASSWORD") - + ap.error( + "Credencials requerides: --user/--password o variables env TAIGA_USERNAME/TAIGA_PASSWORD" + ) # 2. Recuperació del projecte if args.slug: @@ -103,4 +111,4 @@ def main(argv: List[str] | None = None) -> None: # 3. Resultat final print("\n===== RESULTAT =====") - print(f"Nom del projecte : {project['name']}") \ No newline at end of file + print(f"Nom del projecte : {project['name']}") diff --git a/routes/API_publisher/API_event_publisher.py b/routes/API_publisher/API_event_publisher.py index 5efb120..2969921 100644 --- a/routes/API_publisher/API_event_publisher.py +++ b/routes/API_publisher/API_event_publisher.py @@ -5,15 +5,17 @@ logger = logging.getLogger(__name__) -def notify_eval_push(event_type: str,prj: str ,author_login: str , quality_model: str)-> None: - ''' +def notify_eval_push( + event_type: str, prj: str, author_login: str, quality_model: str +) -> None: + """ Function used to notify Component LD_Eval about the event that has been pushed to the database. - ''' - + """ + host = os.getenv("EVAL_HOST", "localhost") port = os.getenv("EVAL_PORT", "5001") - url = f"http://{host}:{port}/api/event" - + url = f"http://{host}:{port}/api/event" + event_data = { "event_type": event_type, "prj": prj, @@ -26,4 +28,4 @@ def notify_eval_push(event_type: str,prj: str ,author_login: str , quality_model resp.raise_for_status() logger.info("LD_Eval responded with %s: %s", resp.status_code, resp.json()) except requests.RequestException as e: - logger.error("Error notifying LD_Eval at %s: %s", url, e) \ No newline at end of file + logger.error("Error notifying LD_Eval at %s: %s", url, e) diff --git a/routes/excel_routes.py b/routes/excel_routes.py index 7cbbc4c..f9af451 100644 --- a/routes/excel_routes.py +++ b/routes/excel_routes.py @@ -9,41 +9,41 @@ logger = logging.getLogger(__name__) - excel_bp = Blueprint("excel_bp", __name__) @excel_bp.route("/webhook/excel", methods=["POST"]) def excel_webhook(): - + logger.info("Received Excel webhook request.") - + # Get the raw JSON payload from the request raw_json = request.get_json() if not raw_json: logger.warning("Excel webhook called without JSON payload.") return {"error": "No JSON received"}, 400 - + # Read the query parameters from the request - prj = request.args.get("prj", type=str) - quality_model = request.args.get("quality_model", type=str) # otional, if not provided, we have to use the default one - + prj = request.args.get("prj", type=str) + quality_model = request.args.get( + "quality_model", type=str + ) # otional, if not provided, we have to use the default one + if not prj: logger.warning("Missing required query param: prj") return jsonify({"error": "prj is required as query parameter"}), 400 - # Parse the raw JSON payload using the parse_excel_event function - parsed_data = parse_excel_event(raw_json, prj, quality_model) + parsed_data = parse_excel_event(raw_json, prj, quality_model) if "error" in parsed_data: return parsed_data, 400 logger.info("Excel webhook request processed successfully.") - # Create the collection name based on the project ID + # Create the collection name based on the project ID collection_name = f"{prj}_sheets" event_name = "sheets_activity" - author_login = '' #username of the author - + author_login = "" # username of the author + coll = get_collection(collection_name) # #COMMUNICATION WITH LD_EVAL USING API @@ -53,13 +53,9 @@ def excel_webhook(): # except Exception as e: # logger.error(f"Error notifying LD_EVAL: {e}") # return {"status": "error", "message": str(e)}, 500 - - + logger.info(f"Inserting Excel activity document for team {prj}") # Insert the parsed data into the MongoDB collection coll.insert_one(parsed_data) - - return jsonify({"status": "OK"}) - - + return jsonify({"status": "OK"}) diff --git a/routes/github_routes.py b/routes/github_routes.py index 899437a..4bf97ae 100644 --- a/routes/github_routes.py +++ b/routes/github_routes.py @@ -12,50 +12,56 @@ github_bp = Blueprint("github_bp", __name__) + @github_bp.route("/webhook/github", methods=["POST"]) def github_webhook(): - + logger.info("Received Github webhook request.") - - # Signature verfication, in the definition of the webhook we must have the same value as in the .env file - secret=GITHUB_SIGNATURE_KEY.encode() + + # Signature verfication, in the definition of the webhook we must have the same value as in the .env file + secret = GITHUB_SIGNATURE_KEY.encode() if not verify_github_signature(request, secret): logger.warning("Invalid Github webhook signature.") - return jsonify({"error": "Invalid Signature"}), 403 - + return jsonify({"error": "Invalid Signature"}), 403 + # Get the raw JSON payload from the request raw_payload = request.get_json() if not raw_payload: logger.warning("Github webhook called without JSON payload.") return {"error": "No JSON received"}, 400 - - + # Read the query parameters from the request - prj = request.args.get("prj", type=str) - quality_model = request.args.get("quality_model", type=str) # otional, if not provided, we have to use the default one + prj = request.args.get("prj", type=str) + quality_model = request.args.get( + "quality_model", type=str + ) # otional, if not provided, we have to use the default one if not prj: logger.warning("Missing required query param: prj") return jsonify({"error": "prj is required as query parameter"}), 400 - - + # We read the event name from the GitHub header, not in the JSON event_name = request.headers.get("X-GitHub-Event") - raw_payload["X-GitHub-Event"] = event_name # Put it in the JSON so parse function sees it + raw_payload["X-GitHub-Event"] = ( + event_name # Put it in the JSON so parse function sees it + ) # Parse the raw JSON payload using the parse_github_event function parsed_data = parse_github_event(raw_payload, prj) - logger.info(f"Github webhook request processed successfully for team {prj}.") - + logger.info(f"Github webhook request processed successfully for team {prj}.") + if parsed_data.get("ignored"): return {"status": "ignored", "event": parsed_data["event"]}, 200 if "error" in parsed_data: return parsed_data, 400 + team_name = parsed_data[ + "team_name" + ] # We wont use this, we will use the external_id instead as its a CENTRALIZED ID + event_label = parsed_data["event"] # This is either "commit" or "issue" + author_login = parsed_data["sender_info"][ + "login" + ] # username of the author of the commit or issue - team_name = parsed_data["team_name"] #We wont use this, we will use the external_id instead as its a CENTRALIZED ID - event_label = parsed_data["event"] #This is either "commit" or "issue" - author_login = parsed_data["sender_info"]["login"] #username of the author of the commit or issue - # Decide the name of the MongoDB collection to write to, depending on the event type if event_label == "commit": collection_name = f"github_{prj}.commits" @@ -63,17 +69,18 @@ def github_webhook(): collection_name = f"{prj}_issues" elif event_label == "pull_request": collection_name = f"github_{prj}.pull_requests" - + coll = get_collection(collection_name) # # COMMUNICATION WITH LD_EVAL USING API - logger.info(f"Notifying LD_EVAL about event: {event_name} for team with external_id: {prj} with quality_model: {quality_model}") + logger.info( + f"Notifying LD_EVAL about event: {event_name} for team with external_id: {prj} with quality_model: {quality_model}" + ) try: notify_eval_push(event_name, prj, author_login, quality_model) except Exception as e: logger.error(f"Error notifying LD_EVAL: {e}") return {"status": "error", "message": str(e)}, 500 - # If it's a commit push, we may have multiple commits. We need to insert each one separately. if "commits" in parsed_data: @@ -97,14 +104,13 @@ def github_webhook(): coll.insert_one(parsed_data) logger.info(f"Inserting in MongoDB Github issue for team {prj}") return {"status": "ok", "message": "Issue inserted"}, 200 - - + elif "pull_request" in parsed_data: parsed_data["prj"] = prj coll.insert_one(parsed_data) logger.info(f"Inserting in MongoDB Github closed pull request for team {prj}") return {"status": "ok", "message": "Pull request inserted"}, 200 - #If its neither a commit or a issue + # If its neither a commit or a issue coll.insert_one(parsed_data) return {"status": "ok", "message": "Stored event doc"}, 200 diff --git a/routes/taiga_routes.py b/routes/taiga_routes.py index e57c5db..17ed231 100644 --- a/routes/taiga_routes.py +++ b/routes/taiga_routes.py @@ -6,40 +6,39 @@ from routes.verify_signature.verify_signature_taiga import verify_taiga_signature import logging -logger = logging.getLogger(__name__) +logger = logging.getLogger(__name__) taiga_bp = Blueprint("taiga_bp", __name__) @taiga_bp.route("/webhook/taiga", methods=["POST"]) def taiga_webhook(): - + logger.info("Received Taiga webhook request.") - - # Signature verfication, in the definition of the webhook we must have the same value as in the .env file - secret=TAIGA_SIGNATURE_KEY.encode() + + # Signature verfication, in the definition of the webhook we must have the same value as in the .env file + secret = TAIGA_SIGNATURE_KEY.encode() if not verify_taiga_signature(request, secret): logger.warning("Invalid Taiga webhook signature.") - return jsonify({"error": "Invalid Signature"}), 403 - + return jsonify({"error": "Invalid Signature"}), 403 + # Get the raw JSON payload from the request raw_payload = request.json if not raw_payload: logger.warning("Taiga webhook called without JSON payload.") return jsonify({"error": "No JSON"}), 400 - # Read the query parameters from the request prj = request.args.get("prj", type=str) - quality_model = request.args.get("quality_model", type=str) # otional, if not provided, we have to use the default one - + quality_model = request.args.get( + "quality_model", type=str + ) # otional, if not provided, we have to use the default one # Get important values from the payload - event_type= raw_payload.get("type","") - action_type= raw_payload.get("action","") - id = raw_payload.get("data",{}).get("id", "") - team_name = raw_payload.get("data",{}).get("project", {}).get("name", "") - + event_type = raw_payload.get("type", "") + action_type = raw_payload.get("action", "") + id = raw_payload.get("data", {}).get("id", "") + team_name = raw_payload.get("data", {}).get("project", {}).get("name", "") # Decide the Mongo collection name to write to, depending on the event type if event_type in ["userstory", "relateduserstory"]: @@ -55,7 +54,7 @@ def taiga_webhook(): coll = get_collection(collection_name) - #Handle the deletion of a document before we parse the payload, to avoid data errors + # Handle the deletion of a document before we parse the payload, to avoid data errors if action_type == "delete": logger.info(f"Deleting document from {collection_name}. ID={id}") if not id: @@ -63,16 +62,16 @@ def taiga_webhook(): coll.delete_one({f"{event_type}_id": id}) logger.info(f"Document with {event_type}={id} has been deleted.") return jsonify({"status": "ok"}), 200 - - - # Parse the raw JSON payload using the parse_taiga_event function + # Parse the raw JSON payload using the parse_taiga_event function parsed_data = parse_taiga_event(raw_payload, prj) logger.info("Taiga webhook request processed successfully.") - author_login = parsed_data["assigned_by"] #username of the author of the commit or issue + author_login = parsed_data[ + "assigned_by" + ] # username of the author of the commit or issue - #If the event is a user story, identify the user story ID and upsert/insert it in the collection + # If the event is a user story, identify the user story ID and upsert/insert it in the collection if event_type in ["userstory", "relateduserstory"]: # UP-SERT user stories in the same collection user_story_id = parsed_data.get("userstory_id") @@ -82,22 +81,16 @@ def taiga_webhook(): logger.info(f"Upserting user story with ID: {user_story_id}") parsed_data["prj"] = prj result = coll.update_one( - {"userstory_id": user_story_id}, - {"$set": parsed_data}, - upsert=True + {"userstory_id": user_story_id}, {"$set": parsed_data}, upsert=True ) logger.info(f"Inserting in MongoDB Taiga userstory for team {prj}") - - - #If the event is a taks , identify the task ID and upsert/insert it in the collection + + # If the event is a taks , identify the task ID and upsert/insert it in the collection elif event_type == "task": - - #if in the parsed data is_closed is true, means the task is closed, so we have to update the points of the user story - #if parsed_data.get("is_closed") == True and parsed_data.get("userstory_is_closed") == True: - - - - + + # if in the parsed data is_closed is true, means the task is closed, so we have to update the points of the user story + # if parsed_data.get("is_closed") == True and parsed_data.get("userstory_is_closed") == True: + coll = get_collection(collection_name) task_id = parsed_data.get("task_id") if not task_id: @@ -107,14 +100,11 @@ def taiga_webhook(): # Upsert instead of insert parsed_data["prj"] = prj result = coll.update_one( - {"task_id": task_id}, - {"$set": parsed_data}, - upsert=True - ) + {"task_id": task_id}, {"$set": parsed_data}, upsert=True + ) logger.info(f"Inserting in MongoDB Taiga task for team {prj}") - - - #If the event is an epic, identify the epic ID and upsert/insert it in the collection + + # If the event is an epic, identify the epic ID and upsert/insert it in the collection elif event_type == "epic": coll = get_collection(collection_name) epic_id = parsed_data.get("epic_id") @@ -125,13 +115,10 @@ def taiga_webhook(): # Upsert instead of insert parsed_data["prj"] = prj result = coll.update_one( - {"epic_id": epic_id}, - {"$set": parsed_data}, - upsert=True - ) + {"epic_id": epic_id}, {"$set": parsed_data}, upsert=True + ) logger.info(f"Inserting in MongoDB Taiga epic for team {prj}") - - + # If the event is an issue, identify the issue ID and upsert/insert it in the collection elif event_type == "issue": coll = get_collection(collection_name) @@ -143,26 +130,23 @@ def taiga_webhook(): parsed_data["prj"] = prj # Upsert instead of insert result = coll.update_one( - {"issue_id": issue_id}, - {"$set": parsed_data}, - upsert=True - ) + {"issue_id": issue_id}, {"$set": parsed_data}, upsert=True + ) logger.info(f"Inserting in MongoDB Taiga issue for team {prj}") - - + else: # If the event is not one of the above, we will insert it as a new document parsed_data["prj"] = prj inserted_id = coll.insert_one(parsed_data).inserted_id - - #COMMUNICATION WITH LD_EVAL USING API - logger.info(f"Notifying LD_EVAL about event: {event_type} for team with external_id: {prj} with quality_model: {quality_model}") + # COMMUNICATION WITH LD_EVAL USING API + logger.info( + f"Notifying LD_EVAL about event: {event_type} for team with external_id: {prj} with quality_model: {quality_model}" + ) try: notify_eval_push(event_type, prj, author_login, quality_model) except Exception as e: logger.error(f"Error notifying LD_EVAL: {e}") return jsonify({"error": "Failed to notify LD_EVAL"}), 500 - - + return jsonify({"status": "ok"}), 200 diff --git a/routes/verify_signature/verify_signature_github.py b/routes/verify_signature/verify_signature_github.py index c170eee..9bc543a 100644 --- a/routes/verify_signature/verify_signature_github.py +++ b/routes/verify_signature/verify_signature_github.py @@ -5,20 +5,18 @@ def verify_github_signature(request, secret): """ Validates the GitHub HMAC signature on the incoming request. - + """ # The signature sent by GitHub, e.g. "sha256=abc123..." signature_header = request.headers.get("X-Hub-Signature-256", "") - + # The raw request body (as bytes), essential for computing HMAC raw_body = request.data - + # Compute our own HMAC sha256 - expected_signature = "sha256=" + hmac.new( - secret, - raw_body, - hashlib.sha256 - ).hexdigest() + expected_signature = ( + "sha256=" + hmac.new(secret, raw_body, hashlib.sha256).hexdigest() + ) # Safely compare the two return hmac.compare_digest(expected_signature, signature_header) diff --git a/routes/verify_signature/verify_signature_taiga.py b/routes/verify_signature/verify_signature_taiga.py index 0ea273e..0a4e9db 100644 --- a/routes/verify_signature/verify_signature_taiga.py +++ b/routes/verify_signature/verify_signature_taiga.py @@ -1,6 +1,7 @@ import hashlib import hmac + def verify_taiga_signature(request, secret): """ Validates the Taiga HMAC-SHA1 signature on the incoming request. diff --git a/tests/conftest.py b/tests/conftest.py index 01c60a7..c378ad9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ """ Shared pytest fixtures for the LD_Connect_Event test suite. """ + import os, json, pytest # ── Set required env vars BEFORE any application module is imported ────────── @@ -16,6 +17,7 @@ # ── Fixtures ───────────────────────────────────────────────────────────────── + @pytest.fixture def sample_credentials_config(tmp_path): """Create a temporary credentials JSON file and return its path.""" @@ -24,14 +26,14 @@ def sample_credentials_config(tmp_path): "github_token": "ghp_FAKETOKEN123", "taiga_user": "tuser", "taiga_password": "tpass", - "teams": ["TeamAlpha", "TeamBeta"] + "teams": ["TeamAlpha", "TeamBeta"], }, "course_b": { "github_token": "ghp_FAKETOKEN456", "taiga_user": "", "taiga_password": "", - "teams": ["TeamGamma"] - } + "teams": ["TeamGamma"], + }, } p = tmp_path / "creds.json" p.write_text(json.dumps(data)) @@ -42,6 +44,7 @@ def sample_credentials_config(tmp_path): def flask_app(): """Create a Flask test application.""" from app import create_app + app = create_app() app.config["TESTING"] = True return app @@ -55,6 +58,7 @@ def client(flask_app): # ── Sample payloads ────────────────────────────────────────────────────────── + @pytest.fixture def github_push_payload(): """Minimal GitHub push webhook payload.""" @@ -67,7 +71,7 @@ def github_push_payload(): "login": "devuser", "url": "https://api.github.com/users/devuser", "type": "User", - "site_admin": False + "site_admin": False, }, "commits": [ { @@ -78,10 +82,10 @@ def github_push_payload(): "author": { "username": "devuser", "name": "Dev User", - "email": "dev@example.com" - } + "email": "dev@example.com", + }, } - ] + ], } @@ -98,15 +102,15 @@ def github_issue_payload(): "login": "issueuser", "url": "https://api.github.com/users/issueuser", "type": "User", - "site_admin": False + "site_admin": False, }, "issue": { "number": 10, "title": "Bug in login", "state": "open", "body": "Login fails with error 500", - "user": {"login": "issueuser", "id": 2} - } + "user": {"login": "issueuser", "id": 2}, + }, } @@ -123,7 +127,7 @@ def github_pr_payload(): "login": "pruser", "url": "https://api.github.com/users/pruser", "type": "User", - "site_admin": False + "site_admin": False, }, "pull_request": { "number": 5, @@ -133,8 +137,8 @@ def github_pr_payload(): "merged": True, "merged_by": {"login": "merger"}, "assignee": {"login": "pruser"}, - "requested_reviewers": [{"login": "reviewer1"}] - } + "requested_reviewers": [{"login": "reviewer1"}], + }, } @@ -162,11 +166,11 @@ def taiga_task_payload(): "created_date": "2025-05-01T00:00:00Z", "modified_date": "2025-06-01T00:00:00Z", "estimated_start": "2025-05-01T00:00:00Z", - "estimated_finish": "2025-06-01T00:00:00Z" + "estimated_finish": "2025-06-01T00:00:00Z", }, "assigned_to": {"username": "dev1"}, - "custom_attributes_values": {"story_points": 5} - } + "custom_attributes_values": {"story_points": 5}, + }, } @@ -191,9 +195,9 @@ def taiga_issue_payload(): "modified_date": "2025-06-10T10:00:00Z", "created_date": "2025-06-01T08:00:00Z", "finished_date": None, - "assigned_to": {"username": "dev2"} + "assigned_to": {"username": "dev2"}, }, - "is_closed": False + "is_closed": False, } @@ -211,9 +215,9 @@ def taiga_epic_payload(): "status": {"name": "New"}, "is_closed": False, "modified_date": "2025-06-10T10:00:00Z", - "created_date": "2025-06-01T08:00:00Z" + "created_date": "2025-06-01T08:00:00Z", }, - "is_closed": False + "is_closed": False, } @@ -242,10 +246,10 @@ def taiga_userstory_payload(): "created_date": "2025-05-01T00:00:00Z", "modified_date": "2025-06-01T00:00:00Z", "estimated_start": "2025-05-01T00:00:00Z", - "estimated_finish": "2025-06-01T00:00:00Z" - } + "estimated_finish": "2025-06-01T00:00:00Z", + }, }, - "is_closed": False + "is_closed": False, } @@ -262,11 +266,11 @@ def taiga_related_userstory_payload(): "id": 300, "subject": "Epic feature", "ref": 1, - "project": {"name": "TestProject"} + "project": {"name": "TestProject"}, }, "finished_date": "2025-07-01T12:00:00Z", - "assigned_to": {"username": "dev1"} - } + "assigned_to": {"username": "dev1"}, + }, } @@ -283,5 +287,5 @@ def excel_payload(): "epic": "Epic 1", "members": ["Alice", "Bob", ""], "memberHours": [3, 2], - "configRange": [0, 0, 0, 0, 5, 0, 0, 0] + "configRange": [0, 0, 0, 0, 5, 0, 0, 0], } diff --git a/tests/test_api_event_publisher.py b/tests/test_api_event_publisher.py index 2270c2f..5df4076 100644 --- a/tests/test_api_event_publisher.py +++ b/tests/test_api_event_publisher.py @@ -1,4 +1,5 @@ """Tests for routes/API_publisher/API_event_publisher.py""" + import pytest from unittest.mock import patch, MagicMock @@ -27,6 +28,7 @@ def test_successful_notification(self, mock_post): def test_network_error_does_not_raise(self, mock_post): import requests as req from routes.API_publisher.API_event_publisher import notify_eval_push + mock_post.side_effect = req.RequestException("Connection refused") # Should not raise — it logs and continues notify_eval_push("push", "P", "u", "qm") diff --git a/tests/test_app.py b/tests/test_app.py index 3394afc..d19953d 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -1,4 +1,5 @@ """Tests for app.py""" + import pytest diff --git a/tests/test_credentials_loader.py b/tests/test_credentials_loader.py index 23173c8..f2daf74 100644 --- a/tests/test_credentials_loader.py +++ b/tests/test_credentials_loader.py @@ -1,4 +1,5 @@ """Tests for config/credentials_loader.py""" + import json, os, pytest from unittest.mock import patch @@ -7,6 +8,7 @@ class TestLoad: def test_load_returns_parsed_json(self, sample_credentials_config): with patch("config.credentials_loader.CONFIG_FILE", sample_credentials_config): from config.credentials_loader import load + data = load() assert "course_a" in data assert data["course_a"]["github_token"] == "ghp_FAKETOKEN123" @@ -16,6 +18,7 @@ def test_load_file_not_found(self, tmp_path): bad_path = str(tmp_path / "nonexistent.json") with patch("config.credentials_loader.CONFIG_FILE", bad_path): from config.credentials_loader import load + with pytest.raises(FileNotFoundError): load() @@ -24,6 +27,7 @@ def test_load_invalid_json(self, tmp_path): p.write_text("{invalid json") with patch("config.credentials_loader.CONFIG_FILE", str(p)): from config.credentials_loader import load + with pytest.raises(json.JSONDecodeError): load() @@ -32,29 +36,34 @@ class TestResolve: def test_resolve_existing_project(self, sample_credentials_config): with patch("config.credentials_loader.CONFIG_FILE", sample_credentials_config): from config.credentials_loader import resolve + token = resolve("TeamAlpha", "github_token") assert token == "ghp_FAKETOKEN123" def test_resolve_second_course(self, sample_credentials_config): with patch("config.credentials_loader.CONFIG_FILE", sample_credentials_config): from config.credentials_loader import resolve + token = resolve("TeamGamma", "github_token") assert token == "ghp_FAKETOKEN456" def test_resolve_project_not_found(self, sample_credentials_config): with patch("config.credentials_loader.CONFIG_FILE", sample_credentials_config): from config.credentials_loader import resolve + with pytest.raises(KeyError, match="NonExistentProject"): resolve("NonExistentProject", "github_token") def test_resolve_field_missing_returns_none(self, sample_credentials_config): with patch("config.credentials_loader.CONFIG_FILE", sample_credentials_config): from config.credentials_loader import resolve + result = resolve("TeamAlpha", "nonexistent_field") assert result is None def test_resolve_empty_string_field(self, sample_credentials_config): with patch("config.credentials_loader.CONFIG_FILE", sample_credentials_config): from config.credentials_loader import resolve + result = resolve("TeamGamma", "taiga_user") assert result == "" diff --git a/tests/test_datetime_utils.py b/tests/test_datetime_utils.py index 358a176..05e0554 100644 --- a/tests/test_datetime_utils.py +++ b/tests/test_datetime_utils.py @@ -1,4 +1,5 @@ """Tests for utils/datetime_utils.py""" + import pytest from utils.datetime_utils import to_madrid_local diff --git a/tests/test_delete_webhooks_github.py b/tests/test_delete_webhooks_github.py index 0b4a4db..fa68bb5 100644 --- a/tests/test_delete_webhooks_github.py +++ b/tests/test_delete_webhooks_github.py @@ -1,4 +1,5 @@ """Tests for utils/webhook_deletion/delete_webhooks_github.py""" + import pytest from unittest.mock import patch, MagicMock @@ -35,7 +36,9 @@ class TestDeleteAllGithubWebhooks: @patch("utils.webhook_deletion.delete_webhooks_github.list_github_hooks") @patch("utils.webhook_deletion.delete_webhooks_github.pymongo.MongoClient") def test_deletes_matching_webhooks(self, mock_mongo, mock_list, mock_delete): - from utils.webhook_deletion.delete_webhooks_github import delete_all_github_webhooks + from utils.webhook_deletion.delete_webhooks_github import ( + delete_all_github_webhooks, + ) # Setup mock MongoDB mock_db = MagicMock() @@ -57,7 +60,9 @@ def test_deletes_matching_webhooks(self, mock_mongo, mock_list, mock_delete): @patch("utils.webhook_deletion.delete_webhooks_github.pymongo.MongoClient") def test_http_error_listing_continues(self, mock_mongo, mock_list): import requests - from utils.webhook_deletion.delete_webhooks_github import delete_all_github_webhooks + from utils.webhook_deletion.delete_webhooks_github import ( + delete_all_github_webhooks, + ) mock_db = MagicMock() mock_db.list_collection_names.return_value = ["github_prj.commits"] diff --git a/tests/test_delete_webhooks_taiga.py b/tests/test_delete_webhooks_taiga.py index 2a1cbd7..434fc6c 100644 --- a/tests/test_delete_webhooks_taiga.py +++ b/tests/test_delete_webhooks_taiga.py @@ -1,4 +1,5 @@ """Tests for utils/webhook_deletion/delete_webhooks_taiga.py""" + import pytest from unittest.mock import patch, MagicMock @@ -35,7 +36,9 @@ class TestDeleteAllTaigaWebhooks: @patch("utils.webhook_deletion.delete_webhooks_taiga.list_taiga_hooks") @patch("utils.webhook_deletion.delete_webhooks_taiga.pymongo.MongoClient") def test_deletes_matching_webhooks(self, mock_mongo, mock_list, mock_delete): - from utils.webhook_deletion.delete_webhooks_taiga import delete_all_taiga_webhooks + from utils.webhook_deletion.delete_webhooks_taiga import ( + delete_all_taiga_webhooks, + ) mock_db = MagicMock() mock_db.list_collection_names.return_value = ["taiga_prj.epics", "other"] @@ -55,7 +58,9 @@ def test_deletes_matching_webhooks(self, mock_mongo, mock_list, mock_delete): @patch("utils.webhook_deletion.delete_webhooks_taiga.pymongo.MongoClient") def test_delete_error_continues(self, mock_mongo, mock_list, mock_delete): import requests - from utils.webhook_deletion.delete_webhooks_taiga import delete_all_taiga_webhooks + from utils.webhook_deletion.delete_webhooks_taiga import ( + delete_all_taiga_webhooks, + ) mock_db = MagicMock() mock_db.list_collection_names.return_value = ["taiga_prj.epics"] diff --git a/tests/test_excel_handler.py b/tests/test_excel_handler.py index 1e20da6..ece9a73 100644 --- a/tests/test_excel_handler.py +++ b/tests/test_excel_handler.py @@ -1,4 +1,5 @@ """Tests for datasources/excel_handler.py""" + import pytest from datasources.excel_handler import parse_excel_event, ACTIVITY_TYPES @@ -44,7 +45,7 @@ def test_empty_members(self): "epic": "", "members": [], "memberHours": [], - "configRange": [] + "configRange": [], } result = parse_excel_event(payload, "TestPrj", "qm1") assert result["members"] == [] @@ -64,7 +65,7 @@ def test_more_members_than_hours(self): "epic": "", "members": ["A", "B"], "memberHours": [1, 2, 3], - "configRange": [] + "configRange": [], } result = parse_excel_event(payload, "P", "qm") assert "hours_A" in result @@ -83,7 +84,7 @@ def test_config_range_shorter_than_activity_types(self): "epic": "", "members": [], "memberHours": [], - "configRange": [10, 20] # Only 2 values for 8 activity types + "configRange": [10, 20], # Only 2 values for 8 activity types } result = parse_excel_event(payload, "P", "qm") assert result["hours_Reunió_d'equip"] == 10 @@ -102,7 +103,7 @@ def test_config_range_with_none_values(self): "epic": "", "members": [], "memberHours": [], - "configRange": [None, 5, None, None, None, None, None, None] + "configRange": [None, 5, None, None, None, None, None, None], } result = parse_excel_event(payload, "P", "qm") assert result["hours_Reunió_d'equip"] == 0 @@ -128,7 +129,7 @@ def test_whitespace_in_members(self): "epic": "", "members": [" Alice ", " ", "Bob"], "memberHours": [1, 2], - "configRange": [] + "configRange": [], } result = parse_excel_event(payload, "P", "qm") assert result["members"] == ["Alice", "Bob"] diff --git a/tests/test_excel_routes.py b/tests/test_excel_routes.py index 8c37dac..6b0fe9f 100644 --- a/tests/test_excel_routes.py +++ b/tests/test_excel_routes.py @@ -1,4 +1,5 @@ """Tests for routes/excel_routes.py""" + import json, pytest from unittest.mock import patch, MagicMock @@ -6,7 +7,9 @@ class TestExcelWebhook: @patch("routes.excel_routes.get_collection") @patch("routes.excel_routes.parse_excel_event") - def test_successful_excel_webhook(self, mock_parse, mock_coll, client, excel_payload): + def test_successful_excel_webhook( + self, mock_parse, mock_coll, client, excel_payload + ): mock_parse.return_value = {"team": "TestPrj", "activity_type": "Dev"} mock_collection = MagicMock() mock_coll.return_value = mock_collection @@ -14,7 +17,7 @@ def test_successful_excel_webhook(self, mock_parse, mock_coll, client, excel_pay resp = client.post( "/webhook/excel?prj=TestPrj&quality_model=default", data=json.dumps(excel_payload), - content_type="application/json" + content_type="application/json", ) assert resp.status_code == 200 data = resp.get_json() @@ -23,9 +26,7 @@ def test_successful_excel_webhook(self, mock_parse, mock_coll, client, excel_pay def test_missing_json_body(self, client): resp = client.post( - "/webhook/excel?prj=TestPrj", - data="", - content_type="application/json" + "/webhook/excel?prj=TestPrj", data="", content_type="application/json" ) assert resp.status_code == 400 @@ -33,7 +34,7 @@ def test_missing_prj_param(self, client, excel_payload): resp = client.post( "/webhook/excel", data=json.dumps(excel_payload), - content_type="application/json" + content_type="application/json", ) assert resp.status_code == 400 data = resp.get_json() @@ -41,12 +42,14 @@ def test_missing_prj_param(self, client, excel_payload): @patch("routes.excel_routes.get_collection") @patch("routes.excel_routes.parse_excel_event") - def test_parse_error_returns_400(self, mock_parse, mock_coll, client, excel_payload): + def test_parse_error_returns_400( + self, mock_parse, mock_coll, client, excel_payload + ): mock_parse.return_value = {"error": "Invalid data"} resp = client.post( "/webhook/excel?prj=TestPrj", data=json.dumps(excel_payload), - content_type="application/json" + content_type="application/json", ) assert resp.status_code == 400 @@ -59,6 +62,6 @@ def test_collection_name_format(self, mock_parse, mock_coll, client, excel_paylo client.post( "/webhook/excel?prj=MyPrj", data=json.dumps(excel_payload), - content_type="application/json" + content_type="application/json", ) mock_coll.assert_called_with("MyPrj_sheets") diff --git a/tests/test_get_taiga_token.py b/tests/test_get_taiga_token.py index 1d3556d..e971839 100644 --- a/tests/test_get_taiga_token.py +++ b/tests/test_get_taiga_token.py @@ -1,4 +1,5 @@ """Tests for utils/taiga_token/get_taiga_token.py""" + import pytest from unittest.mock import patch, MagicMock diff --git a/tests/test_github_api_call.py b/tests/test_github_api_call.py index 20e159e..435e4b7 100644 --- a/tests/test_github_api_call.py +++ b/tests/test_github_api_call.py @@ -1,4 +1,5 @@ """Tests for datasources/requests/github_api_call.py""" + import pytest from unittest.mock import patch, MagicMock @@ -34,7 +35,10 @@ def test_missing_stats_key(self, mock_get, mock_resolve): assert result == {"total": 0, "additions": 0, "deletions": 0} @patch("datasources.requests.github_api_call.resolve", return_value="ghp_TOKEN") - @patch("datasources.requests.github_api_call.requests.get", side_effect=Exception("Network error")) + @patch( + "datasources.requests.github_api_call.requests.get", + side_effect=Exception("Network error"), + ) def test_network_error_returns_zeros(self, mock_get, mock_resolve): from datasources.requests.github_api_call import fetch_commit_stats @@ -44,7 +48,10 @@ def test_network_error_returns_zeros(self, mock_get, mock_resolve): @patch("datasources.requests.github_api_call.resolve", return_value="ghp_TOKEN") @patch("datasources.requests.github_api_call.requests.get") def test_uses_correct_url(self, mock_get, mock_resolve): - from datasources.requests.github_api_call import fetch_commit_stats, GITHUB_API_URL + from datasources.requests.github_api_call import ( + fetch_commit_stats, + GITHUB_API_URL, + ) mock_resp = MagicMock() mock_resp.json.return_value = {"stats": {}} diff --git a/tests/test_github_handler.py b/tests/test_github_handler.py index 8d8fdaa..03d85a0 100644 --- a/tests/test_github_handler.py +++ b/tests/test_github_handler.py @@ -1,30 +1,41 @@ """Tests for datasources/github_handler.py""" + import pytest from unittest.mock import patch, MagicMock class TestParseGithubEvent: - @patch("datasources.github_handler.fetch_commit_stats", return_value={"total": 10, "additions": 7, "deletions": 3}) + @patch( + "datasources.github_handler.fetch_commit_stats", + return_value={"total": 10, "additions": 7, "deletions": 3}, + ) def test_push_event_dispatches(self, mock_stats, github_push_payload): from datasources.github_handler import parse_github_event + result = parse_github_event(github_push_payload, "TestPrj") assert result["event"] == "commit" assert len(result["commits"]) == 1 def test_issue_event_dispatches(self, github_issue_payload): from datasources.github_handler import parse_github_event + result = parse_github_event(github_issue_payload, "TestPrj") assert result["event"] == "issue" assert result["action"] == "opened" - @patch("datasources.github_handler.to_madrid_local", return_value="2025-06-15T14:00:00.000") + @patch( + "datasources.github_handler.to_madrid_local", + return_value="2025-06-15T14:00:00.000", + ) def test_pull_request_event_dispatches(self, mock_tz, github_pr_payload): from datasources.github_handler import parse_github_event + result = parse_github_event(github_pr_payload, "TestPrj") assert result["event"] == "pull_request" def test_unknown_event_returns_ignored(self): from datasources.github_handler import parse_github_event + payload = {"X-GitHub-Event": "deployment"} result = parse_github_event(payload, "TestPrj") assert result["ignored"] is True @@ -32,9 +43,13 @@ def test_unknown_event_returns_ignored(self): class TestParseGithubPushEvent: - @patch("datasources.github_handler.fetch_commit_stats", return_value={"total": 15, "additions": 10, "deletions": 5}) + @patch( + "datasources.github_handler.fetch_commit_stats", + return_value={"total": 15, "additions": 10, "deletions": 5}, + ) def test_basic_push_parsing(self, mock_stats, github_push_payload): from datasources.github_handler import parse_github_push_event + result = parse_github_push_event(github_push_payload, "TestPrj") assert result["event"] == "commit" @@ -43,9 +58,13 @@ def test_basic_push_parsing(self, mock_stats, github_push_payload): assert result["sender_info"]["login"] == "devuser" assert result["sender_info"]["id"] == 1 - @patch("datasources.github_handler.fetch_commit_stats", return_value={"total": 0, "additions": 0, "deletions": 0}) + @patch( + "datasources.github_handler.fetch_commit_stats", + return_value={"total": 0, "additions": 0, "deletions": 0}, + ) def test_commit_details(self, mock_stats, github_push_payload): from datasources.github_handler import parse_github_push_event + result = parse_github_push_event(github_push_payload, "TestPrj") commit = result["commits"][0] @@ -57,102 +76,132 @@ def test_commit_details(self, mock_stats, github_push_payload): assert commit["message_char_count"] == len("fix: resolve task #42 issue") assert commit["message_word_count"] == 5 - @patch("datasources.github_handler.fetch_commit_stats", return_value={"total": 0, "additions": 0, "deletions": 0}) + @patch( + "datasources.github_handler.fetch_commit_stats", + return_value={"total": 0, "additions": 0, "deletions": 0}, + ) def test_task_reference_with_number(self, mock_stats, github_push_payload): from datasources.github_handler import parse_github_push_event + result = parse_github_push_event(github_push_payload, "TestPrj") commit = result["commits"][0] assert commit["task_is_written"] is True assert commit["task_reference"] == 42 - @patch("datasources.github_handler.fetch_commit_stats", return_value={"total": 0, "additions": 0, "deletions": 0}) + @patch( + "datasources.github_handler.fetch_commit_stats", + return_value={"total": 0, "additions": 0, "deletions": 0}, + ) def test_task_reference_catalan(self, mock_stats): from datasources.github_handler import parse_github_push_event + payload = { "organization": {"login": "Org"}, "repository": {"full_name": "Org/repo"}, "sender": {}, - "commits": [{ - "id": "sha1", - "url": "", - "message": "Implementar tasca #99", - "timestamp": "2025-06-15T10:30:00Z", - "author": {"username": "u", "name": "n", "email": "e"} - }] + "commits": [ + { + "id": "sha1", + "url": "", + "message": "Implementar tasca #99", + "timestamp": "2025-06-15T10:30:00Z", + "author": {"username": "u", "name": "n", "email": "e"}, + } + ], } result = parse_github_push_event(payload, "P") commit = result["commits"][0] assert commit["task_is_written"] is True assert commit["task_reference"] == 99 - @patch("datasources.github_handler.fetch_commit_stats", return_value={"total": 0, "additions": 0, "deletions": 0}) + @patch( + "datasources.github_handler.fetch_commit_stats", + return_value={"total": 0, "additions": 0, "deletions": 0}, + ) def test_task_word_without_number(self, mock_stats): from datasources.github_handler import parse_github_push_event + payload = { "organization": {"login": "Org"}, "repository": {"full_name": "Org/repo"}, "sender": {}, - "commits": [{ - "id": "sha1", - "url": "", - "message": "Working on a task", - "timestamp": "2025-06-15T10:30:00Z", - "author": {"username": "u", "name": "n", "email": "e"} - }] + "commits": [ + { + "id": "sha1", + "url": "", + "message": "Working on a task", + "timestamp": "2025-06-15T10:30:00Z", + "author": {"username": "u", "name": "n", "email": "e"}, + } + ], } result = parse_github_push_event(payload, "P") commit = result["commits"][0] assert commit["task_is_written"] is True assert commit["task_reference"] is None - @patch("datasources.github_handler.fetch_commit_stats", return_value={"total": 0, "additions": 0, "deletions": 0}) + @patch( + "datasources.github_handler.fetch_commit_stats", + return_value={"total": 0, "additions": 0, "deletions": 0}, + ) def test_no_task_reference(self, mock_stats): from datasources.github_handler import parse_github_push_event + payload = { "organization": {"login": "Org"}, "repository": {"full_name": "Org/repo"}, "sender": {}, - "commits": [{ - "id": "sha1", - "url": "", - "message": "fix a bug", - "timestamp": "2025-06-15T10:30:00Z", - "author": {"username": "u", "name": "n", "email": "e"} - }] + "commits": [ + { + "id": "sha1", + "url": "", + "message": "fix a bug", + "timestamp": "2025-06-15T10:30:00Z", + "author": {"username": "u", "name": "n", "email": "e"}, + } + ], } result = parse_github_push_event(payload, "P") commit = result["commits"][0] assert commit["task_is_written"] is False assert commit["task_reference"] is None - @patch("datasources.github_handler.fetch_commit_stats", return_value={"total": 0, "additions": 0, "deletions": 0}) + @patch( + "datasources.github_handler.fetch_commit_stats", + return_value={"total": 0, "additions": 0, "deletions": 0}, + ) def test_empty_commits_list(self, mock_stats): from datasources.github_handler import parse_github_push_event + payload = { "organization": {"login": "Org"}, "repository": {"full_name": "Org/repo"}, "sender": {}, - "commits": [] + "commits": [], } result = parse_github_push_event(payload, "P") assert result["commits"] == [] - @patch("datasources.github_handler.fetch_commit_stats", return_value={"total": 20, "additions": 15, "deletions": 5}) + @patch( + "datasources.github_handler.fetch_commit_stats", + return_value={"total": 20, "additions": 15, "deletions": 5}, + ) def test_commit_stats_stored(self, mock_stats, github_push_payload): from datasources.github_handler import parse_github_push_event + result = parse_github_push_event(github_push_payload, "TestPrj") commit = result["commits"][0] assert commit["stats"] == {"total": 20, "additions": 15, "deletions": 5} - @patch("datasources.github_handler.fetch_commit_stats", return_value={"total": 0, "additions": 0, "deletions": 0}) + @patch( + "datasources.github_handler.fetch_commit_stats", + return_value={"total": 0, "additions": 0, "deletions": 0}, + ) def test_missing_organization(self, mock_stats): from datasources.github_handler import parse_github_push_event - payload = { - "repository": {"full_name": "Org/repo"}, - "sender": {}, - "commits": [] - } + + payload = {"repository": {"full_name": "Org/repo"}, "sender": {}, "commits": []} result = parse_github_push_event(payload, "P") assert result["team_name"] == "UnknownTeam" @@ -160,6 +209,7 @@ def test_missing_organization(self, mock_stats): class TestParseGithubIssueEvent: def test_basic_issue_parsing(self, github_issue_payload): from datasources.github_handler import parse_github_issue_event + result = parse_github_issue_event(github_issue_payload, "TestPrj") assert result["event"] == "issue" @@ -169,6 +219,7 @@ def test_basic_issue_parsing(self, github_issue_payload): def test_issue_object(self, github_issue_payload): from datasources.github_handler import parse_github_issue_event + result = parse_github_issue_event(github_issue_payload, "TestPrj") issue = result["issue"] @@ -180,15 +231,20 @@ def test_issue_object(self, github_issue_payload): def test_sender_info(self, github_issue_payload): from datasources.github_handler import parse_github_issue_event + result = parse_github_issue_event(github_issue_payload, "TestPrj") assert result["sender_info"]["login"] == "issueuser" assert result["sender_info"]["id"] == 2 class TestParseGithubPullRequestEvent: - @patch("datasources.github_handler.to_madrid_local", return_value="2025-06-15T14:00:00.000") + @patch( + "datasources.github_handler.to_madrid_local", + return_value="2025-06-15T14:00:00.000", + ) def test_closed_pr_parsed(self, mock_tz, github_pr_payload): from datasources.github_handler import parse_github_pullrequest_event + result = parse_github_pullrequest_event(github_pr_payload, "TestPrj") assert result["event"] == "pull_request" @@ -200,12 +256,13 @@ def test_closed_pr_parsed(self, mock_tz, github_pr_payload): def test_non_closed_pr_ignored(self): from datasources.github_handler import parse_github_pullrequest_event + payload = { "action": "opened", "organization": {"login": "Org"}, "repository": {"full_name": "Org/repo"}, "sender": {}, - "pull_request": {} + "pull_request": {}, } result = parse_github_pullrequest_event(payload, "P") assert result["ignored"] is True @@ -213,6 +270,7 @@ def test_non_closed_pr_ignored(self): @patch("datasources.github_handler.to_madrid_local", return_value="") def test_pr_no_assignee(self, mock_tz): from datasources.github_handler import parse_github_pullrequest_event + payload = { "action": "closed", "organization": {"login": "Org"}, @@ -226,8 +284,8 @@ def test_pr_no_assignee(self, mock_tz): "merged": False, "merged_by": {}, "assignee": None, - "requested_reviewers": [] - } + "requested_reviewers": [], + }, } result = parse_github_pullrequest_event(payload, "P") assert result["reviewers"] == [] diff --git a/tests/test_github_recovery.py b/tests/test_github_recovery.py index e434a29..2388cc8 100644 --- a/tests/test_github_recovery.py +++ b/tests/test_github_recovery.py @@ -1,4 +1,5 @@ """Tests for utils/recovery/github_recovery.py""" + import pytest from unittest.mock import patch, MagicMock from datetime import datetime @@ -7,6 +8,7 @@ class TestParseDt: def test_date_only(self): from utils.recovery.github_recovery import parse_dt + result = parse_dt("2025-06-15") assert result is not None assert result.year == 2025 @@ -15,22 +17,26 @@ def test_date_only(self): def test_datetime_with_time(self): from utils.recovery.github_recovery import parse_dt + result = parse_dt("2025-06-15T14:30") assert result.hour == 14 assert result.minute == 30 def test_none_returns_none(self): from utils.recovery.github_recovery import parse_dt + result = parse_dt(None) assert result is None def test_empty_string_returns_none(self): from utils.recovery.github_recovery import parse_dt + result = parse_dt("") assert result is None def test_timezone_aware(self): from utils.recovery.github_recovery import parse_dt + result = parse_dt("2025-06-15") assert result.tzinfo is not None @@ -39,6 +45,7 @@ class TestGetOrganizationRepos: @patch("utils.recovery.github_recovery.gh_paginated") def test_returns_repo_names(self, mock_pag): from utils.recovery.github_recovery import get_organization_repos + mock_pag.return_value = [ {"name": "repo1"}, {"name": "repo2"}, @@ -88,6 +95,7 @@ def test_multiple_pages(self, mock_get): class TestUpsert: def test_upsert_empty_list(self): from utils.recovery.github_recovery import upsert + mock_coll = MagicMock() result = upsert(mock_coll, [], "sha") assert result == 0 @@ -95,6 +103,7 @@ def test_upsert_empty_list(self): def test_upsert_with_docs(self): from utils.recovery.github_recovery import upsert + mock_coll = MagicMock() mock_result = MagicMock() mock_result.matched_count = 1 @@ -115,22 +124,28 @@ class TestCollectGithub: def test_collect_commits(self, mock_pag, mock_parse, mock_coll, mock_notify): from utils.recovery.github_recovery import collect_github - mock_pag.return_value = [{ - "sha": "abc123", - "url": "https://github.com/Org/repo/commit/abc123", - "commit": { - "message": "fix bug", - "author": {"date": "2025-06-15T10:00:00Z", "name": "Dev", "email": "d@e.com"} - }, - "author": {"login": "dev"} - }] + mock_pag.return_value = [ + { + "sha": "abc123", + "url": "https://github.com/Org/repo/commit/abc123", + "commit": { + "message": "fix bug", + "author": { + "date": "2025-06-15T10:00:00Z", + "name": "Dev", + "email": "d@e.com", + }, + }, + "author": {"login": "dev"}, + } + ] mock_parse.return_value = { "event": "commit", "team_name": "Org", "repo_name": "Org/repo", "sender_info": {"login": "dev"}, - "commits": [{"sha": "abc123", "message": "fix bug"}] + "commits": [{"sha": "abc123", "message": "fix bug"}], } mock_collection = MagicMock() @@ -152,19 +167,16 @@ def test_collect_commits(self, mock_pag, mock_parse, mock_coll, mock_notify): def test_collect_issues(self, mock_pag, mock_parse, mock_coll, mock_notify): from utils.recovery.github_recovery import collect_github - mock_pag.return_value = [{ - "number": 1, - "state": "open", - "user": {"login": "dev"}, - "title": "Bug" - }] + mock_pag.return_value = [ + {"number": 1, "state": "open", "user": {"login": "dev"}, "title": "Bug"} + ] mock_parse.return_value = { "event": "issue", "team_name": "Org", "repo_name": "Org/repo", "sender_info": {"login": "dev"}, - "issue": {"number": 1} + "issue": {"number": 1}, } mock_collection = MagicMock() @@ -184,18 +196,14 @@ def test_collect_issues(self, mock_pag, mock_parse, mock_coll, mock_notify): def test_collect_pull_requests(self, mock_pag, mock_parse, mock_coll, mock_notify): from utils.recovery.github_recovery import collect_github - mock_pag.return_value = [{ - "number": 5, - "user": {"login": "dev"}, - "title": "PR" - }] + mock_pag.return_value = [{"number": 5, "user": {"login": "dev"}, "title": "PR"}] mock_parse.return_value = { "event": "pull_request", "team_name": "Org", "repo_name": "Org/repo", "sender_info": {"login": "dev"}, - "pr_number": 5 + "pr_number": 5, } mock_collection = MagicMock() @@ -205,7 +213,9 @@ def test_collect_pull_requests(self, mock_pag, mock_parse, mock_coll, mock_notif mock_collection.bulk_write.return_value = mock_result mock_coll.return_value = mock_collection - collect_github("Org", "repo", "TestPrj", ["pull_requests"], None, None, "default") + collect_github( + "Org", "repo", "TestPrj", ["pull_requests"], None, None, "default" + ) mock_coll.assert_called_with("github_TestPrj.pull_requests") @patch("utils.recovery.github_recovery.notify_eval_push") @@ -213,5 +223,6 @@ def test_collect_pull_requests(self, mock_pag, mock_parse, mock_coll, mock_notif @patch("utils.recovery.github_recovery.gh_paginated") def test_unsupported_event_logged(self, mock_pag, mock_coll, mock_notify): from utils.recovery.github_recovery import collect_github + # Should not crash on unsupported event collect_github("Org", "repo", "P", ["unsupported"], None, None, "default") diff --git a/tests/test_github_routes.py b/tests/test_github_routes.py index 5cec735..81b3cd7 100644 --- a/tests/test_github_routes.py +++ b/tests/test_github_routes.py @@ -1,4 +1,5 @@ """Tests for routes/github_routes.py""" + import hashlib, hmac, json, pytest from unittest.mock import patch, MagicMock @@ -12,15 +13,15 @@ class TestGithubWebhook: @patch("routes.github_routes.get_collection") @patch("routes.github_routes.parse_github_event") @patch("routes.github_routes.verify_github_signature", return_value=True) - def test_push_event_inserts_commits(self, mock_verify, mock_parse, mock_coll, mock_notify, client): + def test_push_event_inserts_commits( + self, mock_verify, mock_parse, mock_coll, mock_notify, client + ): mock_parse.return_value = { "event": "commit", "team_name": "TestOrg", "repo_name": "TestOrg/repo", "sender_info": {"login": "dev"}, - "commits": [ - {"sha": "abc123", "message": "fix bug"} - ] + "commits": [{"sha": "abc123", "message": "fix bug"}], } mock_collection = MagicMock() mock_coll.return_value = mock_collection @@ -30,10 +31,7 @@ def test_push_event_inserts_commits(self, mock_verify, mock_parse, mock_coll, mo "/webhook/github?prj=TestPrj&quality_model=default", data=body, content_type="application/json", - headers={ - "X-GitHub-Event": "push", - "X-Hub-Signature-256": "sha256=fake" - } + headers={"X-GitHub-Event": "push", "X-Hub-Signature-256": "sha256=fake"}, ) assert resp.status_code == 200 mock_collection.insert_one.assert_called_once() @@ -45,7 +43,7 @@ def test_invalid_signature_returns_403(self, mock_verify, client): "/webhook/github?prj=TestPrj", data=json.dumps({"test": True}), content_type="application/json", - headers={"X-Hub-Signature-256": "sha256=bad"} + headers={"X-Hub-Signature-256": "sha256=bad"}, ) assert resp.status_code == 403 @@ -55,7 +53,7 @@ def test_missing_json_returns_400(self, mock_verify, client): "/webhook/github?prj=TestPrj", data="", content_type="application/json", - headers={"X-Hub-Signature-256": "sha256=x"} + headers={"X-Hub-Signature-256": "sha256=x"}, ) assert resp.status_code == 400 @@ -65,21 +63,23 @@ def test_missing_prj_returns_400(self, mock_verify, client): "/webhook/github", data=json.dumps({"test": True}), content_type="application/json", - headers={"X-Hub-Signature-256": "sha256=x"} + headers={"X-Hub-Signature-256": "sha256=x"}, ) assert resp.status_code == 400 @patch("routes.github_routes.get_collection") @patch("routes.github_routes.parse_github_event") @patch("routes.github_routes.verify_github_signature", return_value=True) - def test_ignored_event_returns_200(self, mock_verify, mock_parse, mock_coll, client): + def test_ignored_event_returns_200( + self, mock_verify, mock_parse, mock_coll, client + ): mock_parse.return_value = {"event": "deployment", "ignored": True} resp = client.post( "/webhook/github?prj=TestPrj", data=json.dumps({"test": True}), content_type="application/json", - headers={"X-GitHub-Event": "deployment", "X-Hub-Signature-256": "sha256=x"} + headers={"X-GitHub-Event": "deployment", "X-Hub-Signature-256": "sha256=x"}, ) assert resp.status_code == 200 data = resp.get_json() @@ -89,13 +89,15 @@ def test_ignored_event_returns_200(self, mock_verify, mock_parse, mock_coll, cli @patch("routes.github_routes.get_collection") @patch("routes.github_routes.parse_github_event") @patch("routes.github_routes.verify_github_signature", return_value=True) - def test_issue_event_inserts_doc(self, mock_verify, mock_parse, mock_coll, mock_notify, client): + def test_issue_event_inserts_doc( + self, mock_verify, mock_parse, mock_coll, mock_notify, client + ): mock_parse.return_value = { "event": "issue", "team_name": "TestOrg", "repo_name": "TestOrg/repo", "sender_info": {"login": "dev"}, - "issue": {"number": 1, "title": "Bug"} + "issue": {"number": 1, "title": "Bug"}, } mock_collection = MagicMock() mock_coll.return_value = mock_collection @@ -104,7 +106,7 @@ def test_issue_event_inserts_doc(self, mock_verify, mock_parse, mock_coll, mock_ "/webhook/github?prj=TestPrj", data=json.dumps({"test": True}), content_type="application/json", - headers={"X-GitHub-Event": "issues", "X-Hub-Signature-256": "sha256=x"} + headers={"X-GitHub-Event": "issues", "X-Hub-Signature-256": "sha256=x"}, ) assert resp.status_code == 200 mock_collection.insert_one.assert_called_once() @@ -113,13 +115,15 @@ def test_issue_event_inserts_doc(self, mock_verify, mock_parse, mock_coll, mock_ @patch("routes.github_routes.get_collection") @patch("routes.github_routes.parse_github_event") @patch("routes.github_routes.verify_github_signature", return_value=True) - def test_pull_request_event(self, mock_verify, mock_parse, mock_coll, mock_notify, client): + def test_pull_request_event( + self, mock_verify, mock_parse, mock_coll, mock_notify, client + ): mock_parse.return_value = { "event": "pull_request", "team_name": "TestOrg", "repo_name": "TestOrg/repo", "sender_info": {"login": "dev"}, - "pull_request": {"number": 5} + "pull_request": {"number": 5}, } mock_collection = MagicMock() mock_coll.return_value = mock_collection @@ -128,7 +132,10 @@ def test_pull_request_event(self, mock_verify, mock_parse, mock_coll, mock_notif "/webhook/github?prj=TestPrj", data=json.dumps({"test": True}), content_type="application/json", - headers={"X-GitHub-Event": "pull_request", "X-Hub-Signature-256": "sha256=x"} + headers={ + "X-GitHub-Event": "pull_request", + "X-Hub-Signature-256": "sha256=x", + }, ) assert resp.status_code == 200 @@ -141,7 +148,7 @@ def test_error_in_parsed_data(self, mock_verify, mock_parse, client): "/webhook/github?prj=TestPrj", data=json.dumps({"test": True}), content_type="application/json", - headers={"X-GitHub-Event": "push", "X-Hub-Signature-256": "sha256=x"} + headers={"X-GitHub-Event": "push", "X-Hub-Signature-256": "sha256=x"}, ) assert resp.status_code == 400 @@ -149,13 +156,15 @@ def test_error_in_parsed_data(self, mock_verify, mock_parse, client): @patch("routes.github_routes.get_collection") @patch("routes.github_routes.parse_github_event") @patch("routes.github_routes.verify_github_signature", return_value=True) - def test_notify_eval_error_returns_500(self, mock_verify, mock_parse, mock_coll, mock_notify, client): + def test_notify_eval_error_returns_500( + self, mock_verify, mock_parse, mock_coll, mock_notify, client + ): mock_parse.return_value = { "event": "commit", "team_name": "T", "repo_name": "T/r", "sender_info": {"login": "u"}, - "commits": [{"sha": "a"}] + "commits": [{"sha": "a"}], } mock_coll.return_value = MagicMock() @@ -163,7 +172,7 @@ def test_notify_eval_error_returns_500(self, mock_verify, mock_parse, mock_coll, "/webhook/github?prj=P", data=json.dumps({"test": True}), content_type="application/json", - headers={"X-GitHub-Event": "push", "X-Hub-Signature-256": "sha256=x"} + headers={"X-GitHub-Event": "push", "X-Hub-Signature-256": "sha256=x"}, ) assert resp.status_code == 500 @@ -171,13 +180,15 @@ def test_notify_eval_error_returns_500(self, mock_verify, mock_parse, mock_coll, @patch("routes.github_routes.get_collection") @patch("routes.github_routes.parse_github_event") @patch("routes.github_routes.verify_github_signature", return_value=True) - def test_collection_name_commit(self, mock_verify, mock_parse, mock_coll, mock_notify, client): + def test_collection_name_commit( + self, mock_verify, mock_parse, mock_coll, mock_notify, client + ): mock_parse.return_value = { "event": "commit", "team_name": "T", "repo_name": "T/r", "sender_info": {"login": "u"}, - "commits": [{"sha": "a"}] + "commits": [{"sha": "a"}], } mock_coll.return_value = MagicMock() @@ -185,7 +196,7 @@ def test_collection_name_commit(self, mock_verify, mock_parse, mock_coll, mock_n "/webhook/github?prj=MyPrj", data=json.dumps({"test": True}), content_type="application/json", - headers={"X-GitHub-Event": "push", "X-Hub-Signature-256": "sha256=x"} + headers={"X-GitHub-Event": "push", "X-Hub-Signature-256": "sha256=x"}, ) mock_coll.assert_called_with("github_MyPrj.commits") @@ -193,13 +204,15 @@ def test_collection_name_commit(self, mock_verify, mock_parse, mock_coll, mock_n @patch("routes.github_routes.get_collection") @patch("routes.github_routes.parse_github_event") @patch("routes.github_routes.verify_github_signature", return_value=True) - def test_collection_name_issue(self, mock_verify, mock_parse, mock_coll, mock_notify, client): + def test_collection_name_issue( + self, mock_verify, mock_parse, mock_coll, mock_notify, client + ): mock_parse.return_value = { "event": "issue", "team_name": "T", "repo_name": "T/r", "sender_info": {"login": "u"}, - "issue": {"number": 1} + "issue": {"number": 1}, } mock_coll.return_value = MagicMock() @@ -207,6 +220,6 @@ def test_collection_name_issue(self, mock_verify, mock_parse, mock_coll, mock_no "/webhook/github?prj=MyPrj", data=json.dumps({"test": True}), content_type="application/json", - headers={"X-GitHub-Event": "issues", "X-Hub-Signature-256": "sha256=x"} + headers={"X-GitHub-Event": "issues", "X-Hub-Signature-256": "sha256=x"}, ) mock_coll.assert_called_with("MyPrj_issues") diff --git a/tests/test_logger_config.py b/tests/test_logger_config.py index 47f512e..ffc96e4 100644 --- a/tests/test_logger_config.py +++ b/tests/test_logger_config.py @@ -1,4 +1,5 @@ """Tests for config/logger_config.py""" + import logging, os, pytest from unittest.mock import patch @@ -12,6 +13,7 @@ def test_setup_logging_default_level(self): with patch.dict(os.environ, {}, clear=False): os.environ.pop("LOG_LEVEL", None) from config.logger_config import setup_logging + setup_logging() assert root.level == logging.INFO @@ -20,6 +22,7 @@ def test_setup_logging_custom_level(self): root.handlers.clear() with patch.dict(os.environ, {"LOG_LEVEL": "DEBUG"}): from config.logger_config import setup_logging + setup_logging() assert root.level == logging.DEBUG @@ -28,6 +31,7 @@ def test_setup_logging_idempotent(self): root = logging.getLogger() root.handlers.clear() from config.logger_config import setup_logging + setup_logging() count_after_first = len(root.handlers) setup_logging() @@ -38,6 +42,7 @@ def test_setup_logging_invalid_level_falls_back(self): root.handlers.clear() with patch.dict(os.environ, {"LOG_LEVEL": "INVALID_LEVEL"}): from config.logger_config import setup_logging + setup_logging() # Should fall back to INFO assert root.level == logging.INFO diff --git a/tests/test_mongo_client.py b/tests/test_mongo_client.py index 85d6fae..deffc7d 100644 --- a/tests/test_mongo_client.py +++ b/tests/test_mongo_client.py @@ -1,4 +1,5 @@ """Tests for database/mongo_client.py""" + import pytest from unittest.mock import patch, MagicMock @@ -11,6 +12,7 @@ def test_get_collection_returns_collection(self): with patch("database.mongo_client.db", mock_db): from database.mongo_client import get_collection + result = get_collection("test_collection") mock_db.__getitem__.assert_called_once_with("test_collection") assert result == mock_collection @@ -28,6 +30,7 @@ def side_effect(name): with patch("database.mongo_client.db", mock_db): from database.mongo_client import get_collection + c1 = get_collection("commits") c2 = get_collection("issues") assert c1 != c2 diff --git a/tests/test_settings.py b/tests/test_settings.py index d4ab09f..383763d 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1,4 +1,5 @@ """Tests for config/settings.py""" + import os, importlib, pytest from unittest.mock import patch @@ -15,8 +16,7 @@ def test_require_env_missing_raises(self): "TAIGA_PASSWORD": "x", "HOME": os.environ.get("HOME", "/tmp"), } - with patch.dict(os.environ, env, clear=True), \ - patch("dotenv.load_dotenv"): + with patch.dict(os.environ, env, clear=True), patch("dotenv.load_dotenv"): with pytest.raises(RuntimeError, match="TAIGA_API_URL"): importlib.reload(settings_mod) # Restore module to working state @@ -38,6 +38,7 @@ def test_mongo_uri_without_credentials(self): with patch.dict(os.environ, env, clear=True): import importlib import config.settings as settings_mod + importlib.reload(settings_mod) assert settings_mod.MONGO_URI == "mongodb://myhost:27017/mydb" @@ -58,6 +59,7 @@ def test_mongo_uri_with_credentials(self): with patch.dict(os.environ, env, clear=True): import importlib import config.settings as settings_mod + importlib.reload(settings_mod) assert "admin:secret" in settings_mod.MONGO_URI assert "authSource=authdb" in settings_mod.MONGO_URI @@ -73,6 +75,7 @@ def test_default_values(self): with patch.dict(os.environ, env, clear=True): import importlib import config.settings as settings_mod + importlib.reload(settings_mod) assert settings_mod.MONGO_HOST == "mongodb" assert settings_mod.MONGO_PORT == "27017" @@ -87,7 +90,6 @@ def test_taiga_auth_url_defaults_to_api_url(self): "TAIGA_PASSWORD": "p", "HOME": os.environ.get("HOME", "/tmp"), } - with patch.dict(os.environ, env, clear=True), \ - patch("dotenv.load_dotenv"): + with patch.dict(os.environ, env, clear=True), patch("dotenv.load_dotenv"): importlib.reload(settings_mod) assert settings_mod.TAIGA_AUTH_URL == "https://custom.taiga.io/api/v1" diff --git a/tests/test_taiga_api_call.py b/tests/test_taiga_api_call.py index 70ae180..48a6be3 100644 --- a/tests/test_taiga_api_call.py +++ b/tests/test_taiga_api_call.py @@ -1,4 +1,5 @@ """Tests for datasources/requests/taiga_api_call.py""" + import pytest from unittest.mock import patch, MagicMock from datetime import datetime, timedelta @@ -8,10 +9,18 @@ class TestMilestoneStats: def setup_method(self): """Clear the cache before each test.""" import datasources.requests.taiga_api_call as mod + mod._CACHE.clear() - @patch("datasources.requests.taiga_api_call.resolve", side_effect=lambda prj, f: {"taiga_user": "u", "taiga_password": "p"}.get(f, "")) - @patch("datasources.requests.taiga_api_call.get_taiga_token", return_value="fake_token") + @patch( + "datasources.requests.taiga_api_call.resolve", + side_effect=lambda prj, f: {"taiga_user": "u", "taiga_password": "p"}.get( + f, "" + ), + ) + @patch( + "datasources.requests.taiga_api_call.get_taiga_token", return_value="fake_token" + ) @patch("datasources.requests.taiga_api_call.requests.get") def test_successful_fetch(self, mock_get, mock_token, mock_resolve): from datasources.requests.taiga_api_call import milestone_stats @@ -23,7 +32,7 @@ def test_successful_fetch(self, mock_get, mock_token, mock_resolve): "total_userstories": 4, "completed_userstories": 2, "total_tasks": 10, - "completed_tasks": 6 + "completed_tasks": 6, } mock_resp.raise_for_status = MagicMock() mock_get.return_value = mock_resp @@ -38,16 +47,25 @@ def test_successful_fetch(self, mock_get, mock_token, mock_resolve): def test_empty_project_id(self): from datasources.requests.taiga_api_call import milestone_stats + result = milestone_stats("", "mile1", "P") assert result == {} def test_empty_milestone_id(self): from datasources.requests.taiga_api_call import milestone_stats + result = milestone_stats("proj1", "", "P") assert result == {} - @patch("datasources.requests.taiga_api_call.resolve", side_effect=lambda prj, f: {"taiga_user": "u", "taiga_password": "p"}.get(f, "")) - @patch("datasources.requests.taiga_api_call.get_taiga_token", return_value="fake_token") + @patch( + "datasources.requests.taiga_api_call.resolve", + side_effect=lambda prj, f: {"taiga_user": "u", "taiga_password": "p"}.get( + f, "" + ), + ) + @patch( + "datasources.requests.taiga_api_call.get_taiga_token", return_value="fake_token" + ) @patch("datasources.requests.taiga_api_call.requests.get") def test_caching(self, mock_get, mock_token, mock_resolve): from datasources.requests.taiga_api_call import milestone_stats @@ -59,7 +77,7 @@ def test_caching(self, mock_get, mock_token, mock_resolve): "total_userstories": 1, "completed_userstories": 0, "total_tasks": 1, - "completed_tasks": 0 + "completed_tasks": 0, } mock_resp.raise_for_status = MagicMock() mock_get.return_value = mock_resp @@ -70,7 +88,10 @@ def test_caching(self, mock_get, mock_token, mock_resolve): milestone_stats("p1", "m1", "P") assert mock_get.call_count == 1 # Only called once due to caching - @patch("datasources.requests.taiga_api_call.resolve", side_effect=lambda prj, f: {"taiga_user": "", "taiga_password": ""}.get(f, "")) + @patch( + "datasources.requests.taiga_api_call.resolve", + side_effect=lambda prj, f: {"taiga_user": "", "taiga_password": ""}.get(f, ""), + ) @patch("datasources.requests.taiga_api_call.requests.get") def test_no_credentials_no_auth_header(self, mock_get, mock_resolve): from datasources.requests.taiga_api_call import milestone_stats @@ -82,7 +103,7 @@ def test_no_credentials_no_auth_header(self, mock_get, mock_resolve): "total_userstories": 0, "completed_userstories": 0, "total_tasks": 0, - "completed_tasks": 0 + "completed_tasks": 0, } mock_resp.raise_for_status = MagicMock() mock_get.return_value = mock_resp @@ -91,7 +112,12 @@ def test_no_credentials_no_auth_header(self, mock_get, mock_resolve): headers = mock_get.call_args[1]["headers"] assert "Authorization" not in headers - @patch("datasources.requests.taiga_api_call.resolve", side_effect=lambda prj, f: {"taiga_user": "u", "taiga_password": "p"}.get(f, "")) + @patch( + "datasources.requests.taiga_api_call.resolve", + side_effect=lambda prj, f: {"taiga_user": "u", "taiga_password": "p"}.get( + f, "" + ), + ) @patch("datasources.requests.taiga_api_call.get_taiga_token", return_value="tok") @patch("datasources.requests.taiga_api_call.requests.get") def test_http_error_returns_zeros(self, mock_get, mock_token, mock_resolve): diff --git a/tests/test_taiga_auth.py b/tests/test_taiga_auth.py index 7a2f53c..e227255 100644 --- a/tests/test_taiga_auth.py +++ b/tests/test_taiga_auth.py @@ -1,4 +1,5 @@ """Tests for utils/taiga_token/taiga_auth.py""" + import time, pytest from unittest.mock import patch, MagicMock @@ -7,6 +8,7 @@ class TestGetTaigaToken: def setup_method(self): """Clear the token cache before each test.""" import utils.taiga_token.taiga_auth as mod + mod._TOKENS.clear() @patch("utils.taiga_token.taiga_auth.requests.post") diff --git a/tests/test_taiga_handler.py b/tests/test_taiga_handler.py index 297d376..9b2ed79 100644 --- a/tests/test_taiga_handler.py +++ b/tests/test_taiga_handler.py @@ -1,4 +1,5 @@ """Tests for datasources/taiga_handler.py""" + import pytest from unittest.mock import patch, MagicMock @@ -7,32 +8,38 @@ class TestParseTaigaEvent: @patch("datasources.taiga_handler.milestone_stats", return_value={}) def test_task_event(self, mock_ms, taiga_task_payload): from datasources.taiga_handler import parse_taiga_event + result = parse_taiga_event(taiga_task_payload, "TestPrj") assert result["event_type"] == "task" def test_issue_event(self, taiga_issue_payload): from datasources.taiga_handler import parse_taiga_event + result = parse_taiga_event(taiga_issue_payload, "TestPrj") assert result["event_type"] == "issue" def test_epic_event(self, taiga_epic_payload): from datasources.taiga_handler import parse_taiga_event + result = parse_taiga_event(taiga_epic_payload, "TestPrj") assert result["event_type"] == "epic" @patch("datasources.taiga_handler.milestone_stats", return_value={}) def test_userstory_event(self, mock_ms, taiga_userstory_payload): from datasources.taiga_handler import parse_taiga_event + result = parse_taiga_event(taiga_userstory_payload, "TestPrj") assert result["event_type"] == "userstory" def test_relateduserstory_event(self, taiga_related_userstory_payload): from datasources.taiga_handler import parse_taiga_event + result = parse_taiga_event(taiga_related_userstory_payload, "TestPrj") assert result["event_type"] == "relateduserstory" def test_unsupported_event(self): from datasources.taiga_handler import parse_taiga_event + payload = {"type": "unknown_type"} result = parse_taiga_event(payload, "P") assert result["event"] == "unknown_type" @@ -42,6 +49,7 @@ def test_unsupported_event(self): class TestParseTaigaIssueEvent: def test_basic_issue_parsing(self, taiga_issue_payload): from datasources.taiga_handler import parse_taiga_issue_event + result = parse_taiga_issue_event(taiga_issue_payload, "TestPrj") assert result["event_type"] == "issue" @@ -58,6 +66,7 @@ def test_basic_issue_parsing(self, taiga_issue_payload): def test_issue_assigned_to_none(self): from datasources.taiga_handler import parse_taiga_issue_event + payload = { "type": "issue", "action": "create", @@ -75,9 +84,9 @@ def test_issue_assigned_to_none(self): "modified_date": "", "created_date": "", "finished_date": "", - "assigned_to": None + "assigned_to": None, }, - "is_closed": False + "is_closed": False, } result = parse_taiga_issue_event(payload, "P") assert result["assigned_to"] is None @@ -86,6 +95,7 @@ def test_issue_assigned_to_none(self): class TestParseTaigaEpicEvent: def test_basic_epic_parsing(self, taiga_epic_payload): from datasources.taiga_handler import parse_taiga_epic_event + result = parse_taiga_epic_event(taiga_epic_payload, "TestPrj") assert result["epic_id"] == 300 @@ -97,9 +107,13 @@ def test_basic_epic_parsing(self, taiga_epic_payload): class TestParseTaigaTaskEvent: - @patch("datasources.taiga_handler.milestone_stats", return_value={"milestone_total_points": 10}) + @patch( + "datasources.taiga_handler.milestone_stats", + return_value={"milestone_total_points": 10}, + ) def test_basic_task_parsing(self, mock_ms, taiga_task_payload): from datasources.taiga_handler import parse_taiga_task_event + result = parse_taiga_task_event(taiga_task_payload, "TestPrj") assert result["event_type"] == "task" @@ -118,6 +132,7 @@ def test_basic_task_parsing(self, mock_ms, taiga_task_payload): @patch("datasources.taiga_handler.milestone_stats", return_value={}) def test_task_assigned_to_none(self, mock_ms): from datasources.taiga_handler import parse_taiga_task_event + payload = { "type": "task", "action": "create", @@ -132,12 +147,18 @@ def test_task_assigned_to_none(self, mock_ms): "modified_date": "", "finished_date": "", "ref": 1, - "milestone": {"id": 1, "name": "S1", "closed": False, - "created_date": "", "modified_date": "", - "estimated_start": "", "estimated_finish": ""}, + "milestone": { + "id": 1, + "name": "S1", + "closed": False, + "created_date": "", + "modified_date": "", + "estimated_start": "", + "estimated_finish": "", + }, "assigned_to": None, - "custom_attributes_values": None - } + "custom_attributes_values": None, + }, } result = parse_taiga_task_event(payload, "P") assert result["assigned_to"] is None @@ -145,9 +166,13 @@ def test_task_assigned_to_none(self, mock_ms): class TestParseTaigaUserstoryEvent: - @patch("datasources.taiga_handler.milestone_stats", return_value={"milestone_total_points": 20}) + @patch( + "datasources.taiga_handler.milestone_stats", + return_value={"milestone_total_points": 20}, + ) def test_basic_userstory_parsing(self, mock_ms, taiga_userstory_payload): from datasources.taiga_handler import parse_taiga_userstory_event + result = parse_taiga_userstory_event(taiga_userstory_payload, "TestPrj") assert result["event_type"] == "userstory" @@ -161,6 +186,7 @@ def test_basic_userstory_parsing(self, mock_ms, taiga_userstory_payload): @patch("datasources.taiga_handler.milestone_stats", return_value={}) def test_userstory_no_pattern(self, mock_ms): from datasources.taiga_handler import parse_taiga_userstory_event + payload = { "type": "userstory", "action": "create", @@ -175,9 +201,9 @@ def test_userstory_no_pattern(self, mock_ms): "description": "Just a simple description", "custom_attributes_values": {}, "points": [], - "milestone": None + "milestone": None, }, - "is_closed": False + "is_closed": False, } result = parse_taiga_userstory_event(payload, "P") assert result["pattern"] is False @@ -189,6 +215,7 @@ def test_userstory_custom_attributes_none(self, mock_ms): """When custom_attributes_values is None, the code crashes on .get('Priority'). This is a known bug in the source. Test with empty dict instead.""" from datasources.taiga_handler import parse_taiga_userstory_event + payload = { "type": "userstory", "action": "create", @@ -203,9 +230,9 @@ def test_userstory_custom_attributes_none(self, mock_ms): "description": "", "custom_attributes_values": {}, "points": [{"value": None}, {"value": 3}], - "milestone": None + "milestone": None, }, - "is_closed": False + "is_closed": False, } result = parse_taiga_userstory_event(payload, "P") assert result["custom_attributes"] == {} @@ -216,7 +243,10 @@ def test_userstory_custom_attributes_none(self, mock_ms): class TestParseTaigaRelatedUserstoryEvent: def test_basic_related_userstory_parsing(self, taiga_related_userstory_payload): from datasources.taiga_handler import parse_taiga_related_userstory_event - result = parse_taiga_related_userstory_event(taiga_related_userstory_payload, "TestPrj") + + result = parse_taiga_related_userstory_event( + taiga_related_userstory_payload, "TestPrj" + ) assert result["event_type"] == "relateduserstory" assert result["id"] == 400 diff --git a/tests/test_taiga_recovery.py b/tests/test_taiga_recovery.py index 5bb7d38..ab8c3d4 100644 --- a/tests/test_taiga_recovery.py +++ b/tests/test_taiga_recovery.py @@ -1,4 +1,5 @@ """Tests for utils/recovery/taiga_recovery.py""" + import pytest from unittest.mock import patch, MagicMock @@ -6,6 +7,7 @@ class TestParseDt: def test_date_only(self): from utils.recovery.taiga_recovery import parse_dt + result = parse_dt("2025-06-15") assert result.year == 2025 assert result.month == 6 @@ -13,6 +15,7 @@ def test_date_only(self): def test_datetime_with_time(self): from utils.recovery.taiga_recovery import parse_dt + result = parse_dt("2025-06-15T14:30") assert result.hour == 14 assert result.minute == 30 @@ -22,6 +25,7 @@ class TestGetProjectIdBySlug: @patch("utils.recovery.taiga_recovery.requests.get") def test_success(self, mock_get): from utils.recovery.taiga_recovery import get_project_id_by_slug + mock_resp = MagicMock() mock_resp.status_code = 200 mock_resp.json.return_value = {"id": 42} @@ -33,6 +37,7 @@ def test_success(self, mock_get): @patch("utils.recovery.taiga_recovery.requests.get") def test_private_project_exits(self, mock_get): from utils.recovery.taiga_recovery import get_project_id_by_slug + mock_resp = MagicMock() mock_resp.status_code = 401 mock_get.return_value = mock_resp @@ -43,6 +48,7 @@ def test_private_project_exits(self, mock_get): @patch("utils.recovery.taiga_recovery.requests.get") def test_not_found_exits(self, mock_get): from utils.recovery.taiga_recovery import get_project_id_by_slug + mock_resp = MagicMock() mock_resp.status_code = 404 mock_get.return_value = mock_resp @@ -54,6 +60,7 @@ def test_not_found_exits(self, mock_get): def test_other_error_raises(self, mock_get): import requests from utils.recovery.taiga_recovery import get_project_id_by_slug + mock_resp = MagicMock() mock_resp.status_code = 500 mock_resp.raise_for_status.side_effect = requests.HTTPError("500") @@ -67,6 +74,7 @@ class TestGetUsernameId: @patch("utils.recovery.taiga_recovery.requests.get") def test_returns_id(self, mock_get): from utils.recovery.taiga_recovery import get_username_id + mock_resp = MagicMock() mock_resp.json.return_value = {"id": 123} mock_resp.raise_for_status = MagicMock() @@ -81,10 +89,11 @@ class TestGetProjectIdByUsernameId: @patch("utils.recovery.taiga_recovery.requests.get") def test_project_found(self, mock_get, mock_uid): from utils.recovery.taiga_recovery import get_project_id_by_username_id + mock_resp = MagicMock() mock_resp.json.return_value = [ {"name": "Other Project", "id": 10}, - {"name": "My Project", "id": 42} + {"name": "My Project", "id": 42}, ] mock_resp.raise_for_status = MagicMock() mock_get.return_value = mock_resp @@ -96,6 +105,7 @@ def test_project_found(self, mock_get, mock_uid): @patch("utils.recovery.taiga_recovery.requests.get") def test_project_not_found_exits(self, mock_get, mock_uid): from utils.recovery.taiga_recovery import get_project_id_by_username_id + mock_resp = MagicMock() mock_resp.json.return_value = [] mock_resp.raise_for_status = MagicMock() @@ -109,6 +119,7 @@ class TestFetchEntities: @patch("utils.recovery.taiga_recovery.requests.get") def test_fetch_tasks(self, mock_get): from utils.recovery.taiga_recovery import fetch_entities + mock_resp = MagicMock() mock_resp.json.return_value = [{"id": 1}, {"id": 2}] mock_resp.raise_for_status = MagicMock() @@ -119,12 +130,14 @@ def test_fetch_tasks(self, mock_get): def test_unsupported_entity_raises(self): from utils.recovery.taiga_recovery import fetch_entities + with pytest.raises(ValueError, match="Not supported"): fetch_entities("wiki", 42) @patch("utils.recovery.taiga_recovery.requests.get") def test_fetch_with_date_filters(self, mock_get): from utils.recovery.taiga_recovery import fetch_entities, parse_dt + mock_resp = MagicMock() mock_resp.json.return_value = [] mock_resp.raise_for_status = MagicMock() @@ -142,11 +155,13 @@ def test_fetch_with_date_filters(self, mock_get): class TestUpsert: def test_empty(self): from utils.recovery.taiga_recovery import upsert + result = upsert(MagicMock(), [], "id") assert result == 0 def test_with_docs(self): from utils.recovery.taiga_recovery import upsert + mock_coll = MagicMock() mock_result = MagicMock() mock_result.matched_count = 2 @@ -160,6 +175,7 @@ def test_with_docs(self): class TestConverters: def test_task_from_api(self): from utils.recovery.taiga_recovery import task_from_api + j = { "id": 100, "subject": "Task A", @@ -174,7 +190,7 @@ def test_task_from_api(self): "status_extra_info": {"is_closed": False, "name": "New"}, "assigned_to_extra_info": {"username": "dev1"}, "custom_attributes_values": {"sp": 5}, - "project_extra_info": {"name": "TestProject", "id": 1} + "project_extra_info": {"name": "TestProject", "id": 1}, } doc = task_from_api(j, "TestPrj") assert doc["task_id"] == 100 @@ -185,6 +201,7 @@ def test_task_from_api(self): def test_issue_from_api(self): from utils.recovery.taiga_recovery import issue_from_api + j = { "id": 200, "subject": "Bug", @@ -198,7 +215,7 @@ def test_issue_from_api(self): "severity_extra_info": {"name": "Normal"}, "type_extra_info": {"name": "Bug"}, "assigned_to_extra_info": None, - "project_extra_info": {"name": "TestProject"} + "project_extra_info": {"name": "TestProject"}, } doc = issue_from_api(j, "TestPrj") assert doc["issue_id"] == 200 @@ -207,13 +224,14 @@ def test_issue_from_api(self): def test_epic_from_api(self): from utils.recovery.taiga_recovery import epic_from_api + j = { "id": 300, "subject": "Epic", "created_date": "2025-06-01", "modified_date": "2025-06-10", "status_extra_info": {"is_closed": False, "name": "New"}, - "project_extra_info": {"name": "TestProject", "id": 1} + "project_extra_info": {"name": "TestProject", "id": 1}, } doc = epic_from_api(j, "TestPrj") assert doc["epic_id"] == 300 @@ -221,6 +239,7 @@ def test_epic_from_api(self): def test_userstory_from_api(self): from utils.recovery.taiga_recovery import userstory_from_api + j = { "id": 400, "subject": "User login", @@ -229,14 +248,17 @@ def test_userstory_from_api(self): "modified_date": "2025-06-10", "status_extra_info": {"is_closed": False, "name": "New"}, "milestone": 10, - "milestone_extra_info": {"name": "Sprint 1", "closed": False, - "created_date": "2025-05-01", - "modified_date": "2025-06-01", - "estimated_start": "2025-05-01", - "estimated_finish": "2025-06-01"}, + "milestone_extra_info": { + "name": "Sprint 1", + "closed": False, + "created_date": "2025-05-01", + "modified_date": "2025-06-01", + "estimated_start": "2025-05-01", + "estimated_finish": "2025-06-01", + }, "custom_attributes_values": {"Priority": "High"}, "points": [{"value": 3}, {"value": 5}], - "project_extra_info": {"name": "TestProject"} + "project_extra_info": {"name": "TestProject"}, } doc = userstory_from_api(j, "TestPrj") assert doc["userstory_id"] == 400 @@ -246,6 +268,7 @@ def test_userstory_from_api(self): def test_userstory_no_pattern(self): from utils.recovery.taiga_recovery import userstory_from_api + j = { "id": 401, "subject": "S", @@ -257,7 +280,7 @@ def test_userstory_no_pattern(self): "milestone_extra_info": None, "custom_attributes_values": None, "points": "", - "project_extra_info": {"name": "P"} + "project_extra_info": {"name": "P"}, } doc = userstory_from_api(j, "P") assert doc["pattern"] is False @@ -271,8 +294,11 @@ class TestMain: @patch("utils.recovery.taiga_recovery.get_collection") @patch("utils.recovery.taiga_recovery.fetch_entities", return_value=[]) @patch("utils.recovery.taiga_recovery.get_project_id_by_slug", return_value=42) - def test_main_runs(self, mock_slug, mock_fetch, mock_coll, mock_upsert, mock_notify): + def test_main_runs( + self, mock_slug, mock_fetch, mock_coll, mock_upsert, mock_notify + ): from utils.recovery.taiga_recovery import main + main(["--slug", "test-slug", "--prj", "TestPrj", "--events", "task"]) mock_slug.assert_called_once_with("test-slug") mock_fetch.assert_called_once() @@ -282,7 +308,21 @@ def test_main_runs(self, mock_slug, mock_fetch, mock_coll, mock_upsert, mock_not @patch("utils.recovery.taiga_recovery.get_collection") @patch("utils.recovery.taiga_recovery.fetch_entities", return_value=[]) @patch("utils.recovery.taiga_recovery.get_project_id_by_slug", return_value=42) - def test_main_with_dates(self, mock_slug, mock_fetch, mock_coll, mock_upsert, mock_notify): + def test_main_with_dates( + self, mock_slug, mock_fetch, mock_coll, mock_upsert, mock_notify + ): from utils.recovery.taiga_recovery import main - main(["--slug", "s", "--prj", "P", "--from-date", "2025-01-01", "--to-date", "2025-12-31"]) + + main( + [ + "--slug", + "s", + "--prj", + "P", + "--from-date", + "2025-01-01", + "--to-date", + "2025-12-31", + ] + ) mock_fetch.assert_called() diff --git a/tests/test_taiga_routes.py b/tests/test_taiga_routes.py index 0999f72..228465f 100644 --- a/tests/test_taiga_routes.py +++ b/tests/test_taiga_routes.py @@ -1,4 +1,5 @@ """Tests for routes/taiga_routes.py""" + import hashlib, hmac, json, pytest from unittest.mock import patch, MagicMock @@ -12,19 +13,27 @@ def _post(self, client, payload, prj="TestPrj", quality_model="default"): f"/webhook/taiga?prj={prj}&quality_model={quality_model}", data=body, content_type="application/json", - headers={"X-TAIGA-WEBHOOK-SIGNATURE": sig} + headers={"X-TAIGA-WEBHOOK-SIGNATURE": sig}, ) @patch("routes.taiga_routes.notify_eval_push") @patch("routes.taiga_routes.get_collection") @patch("routes.taiga_routes.parse_taiga_event") @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) - def test_task_create_upserts(self, mock_verify, mock_parse, mock_coll, mock_notify, client, taiga_task_payload): + def test_task_create_upserts( + self, + mock_verify, + mock_parse, + mock_coll, + mock_notify, + client, + taiga_task_payload, + ): mock_parse.return_value = { "event_type": "task", "task_id": 100, "assigned_by": "u", - "subject": "S" + "subject": "S", } mock_collection = MagicMock() mock_coll.return_value = mock_collection @@ -40,7 +49,7 @@ def test_invalid_signature_403(self, mock_verify, client, taiga_task_payload): "/webhook/taiga?prj=P", data=json.dumps(taiga_task_payload), content_type="application/json", - headers={"X-TAIGA-WEBHOOK-SIGNATURE": "bad"} + headers={"X-TAIGA-WEBHOOK-SIGNATURE": "bad"}, ) assert resp.status_code == 403 @@ -50,18 +59,22 @@ def test_missing_json_400(self, mock_verify, client): "/webhook/taiga?prj=P", data="", content_type="application/json", - headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"} + headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"}, ) assert resp.status_code == 400 @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) def test_unsupported_type_ignored(self, mock_verify, client): - payload = {"type": "wiki", "action": "create", "data": {"id": 1, "project": {"name": "P"}}} + payload = { + "type": "wiki", + "action": "create", + "data": {"id": 1, "project": {"name": "P"}}, + } resp = client.post( "/webhook/taiga?prj=P", data=json.dumps(payload), content_type="application/json", - headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"} + headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"}, ) assert resp.status_code == 200 data = resp.get_json() @@ -74,7 +87,7 @@ def test_delete_action(self, mock_verify, mock_coll, client): "type": "task", "action": "delete", "data": {"id": 99, "project": {"name": "P"}}, - "by": {"username": "u"} + "by": {"username": "u"}, } mock_collection = MagicMock() mock_coll.return_value = mock_collection @@ -83,7 +96,7 @@ def test_delete_action(self, mock_verify, mock_coll, client): "/webhook/taiga?prj=P", data=json.dumps(payload), content_type="application/json", - headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"} + headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"}, ) assert resp.status_code == 200 mock_collection.delete_one.assert_called_once_with({"task_id": 99}) @@ -95,7 +108,7 @@ def test_delete_no_id_returns_400(self, mock_verify, mock_coll, client): "type": "task", "action": "delete", "data": {"id": "", "project": {"name": "P"}}, - "by": {"username": "u"} + "by": {"username": "u"}, } mock_coll.return_value = MagicMock() @@ -103,7 +116,7 @@ def test_delete_no_id_returns_400(self, mock_verify, mock_coll, client): "/webhook/taiga?prj=P", data=json.dumps(payload), content_type="application/json", - headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"} + headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"}, ) assert resp.status_code == 400 @@ -111,11 +124,19 @@ def test_delete_no_id_returns_400(self, mock_verify, mock_coll, client): @patch("routes.taiga_routes.get_collection") @patch("routes.taiga_routes.parse_taiga_event") @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) - def test_userstory_upsert(self, mock_verify, mock_parse, mock_coll, mock_notify, client, taiga_userstory_payload): + def test_userstory_upsert( + self, + mock_verify, + mock_parse, + mock_coll, + mock_notify, + client, + taiga_userstory_payload, + ): mock_parse.return_value = { "event_type": "userstory", "userstory_id": 400, - "assigned_by": "u" + "assigned_by": "u", } mock_collection = MagicMock() mock_coll.return_value = mock_collection @@ -128,11 +149,19 @@ def test_userstory_upsert(self, mock_verify, mock_parse, mock_coll, mock_notify, @patch("routes.taiga_routes.get_collection") @patch("routes.taiga_routes.parse_taiga_event") @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) - def test_epic_upsert(self, mock_verify, mock_parse, mock_coll, mock_notify, client, taiga_epic_payload): + def test_epic_upsert( + self, + mock_verify, + mock_parse, + mock_coll, + mock_notify, + client, + taiga_epic_payload, + ): mock_parse.return_value = { "event_type": "epic", "epic_id": 300, - "assigned_by": "u" + "assigned_by": "u", } mock_collection = MagicMock() mock_coll.return_value = mock_collection @@ -145,11 +174,19 @@ def test_epic_upsert(self, mock_verify, mock_parse, mock_coll, mock_notify, clie @patch("routes.taiga_routes.get_collection") @patch("routes.taiga_routes.parse_taiga_event") @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) - def test_issue_upsert(self, mock_verify, mock_parse, mock_coll, mock_notify, client, taiga_issue_payload): + def test_issue_upsert( + self, + mock_verify, + mock_parse, + mock_coll, + mock_notify, + client, + taiga_issue_payload, + ): mock_parse.return_value = { "event_type": "issue", "issue_id": 200, - "assigned_by": "u" + "assigned_by": "u", } mock_collection = MagicMock() mock_coll.return_value = mock_collection @@ -162,11 +199,19 @@ def test_issue_upsert(self, mock_verify, mock_parse, mock_coll, mock_notify, cli @patch("routes.taiga_routes.get_collection") @patch("routes.taiga_routes.parse_taiga_event") @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) - def test_notify_eval_error_returns_500(self, mock_verify, mock_parse, mock_coll, mock_notify, client, taiga_task_payload): + def test_notify_eval_error_returns_500( + self, + mock_verify, + mock_parse, + mock_coll, + mock_notify, + client, + taiga_task_payload, + ): mock_parse.return_value = { "event_type": "task", "task_id": 100, - "assigned_by": "u" + "assigned_by": "u", } mock_coll.return_value = MagicMock() @@ -177,17 +222,19 @@ def test_notify_eval_error_returns_500(self, mock_verify, mock_parse, mock_coll, @patch("routes.taiga_routes.get_collection") @patch("routes.taiga_routes.parse_taiga_event") @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) - def test_userstory_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, mock_notify, client): + def test_userstory_no_id_returns_400( + self, mock_verify, mock_parse, mock_coll, mock_notify, client + ): payload = { "type": "userstory", "action": "create", "by": {"username": "u"}, - "data": {"id": 1, "project": {"id": 1, "name": "P"}} + "data": {"id": 1, "project": {"id": 1, "name": "P"}}, } mock_parse.return_value = { "event_type": "userstory", "userstory_id": None, - "assigned_by": "u" + "assigned_by": "u", } mock_coll.return_value = MagicMock() @@ -195,7 +242,7 @@ def test_userstory_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, m "/webhook/taiga?prj=P", data=json.dumps(payload), content_type="application/json", - headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"} + headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"}, ) assert resp.status_code == 400 @@ -203,17 +250,19 @@ def test_userstory_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, m @patch("routes.taiga_routes.get_collection") @patch("routes.taiga_routes.parse_taiga_event") @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) - def test_task_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, mock_notify, client): + def test_task_no_id_returns_400( + self, mock_verify, mock_parse, mock_coll, mock_notify, client + ): payload = { "type": "task", "action": "create", "by": {"username": "u"}, - "data": {"id": 1, "project": {"id": 1, "name": "P"}} + "data": {"id": 1, "project": {"id": 1, "name": "P"}}, } mock_parse.return_value = { "event_type": "task", "task_id": None, - "assigned_by": "u" + "assigned_by": "u", } mock_coll.return_value = MagicMock() @@ -221,7 +270,7 @@ def test_task_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, mock_n "/webhook/taiga?prj=P", data=json.dumps(payload), content_type="application/json", - headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"} + headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"}, ) assert resp.status_code == 400 @@ -229,17 +278,19 @@ def test_task_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, mock_n @patch("routes.taiga_routes.get_collection") @patch("routes.taiga_routes.parse_taiga_event") @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) - def test_epic_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, mock_notify, client): + def test_epic_no_id_returns_400( + self, mock_verify, mock_parse, mock_coll, mock_notify, client + ): payload = { "type": "epic", "action": "create", "by": {"username": "u"}, - "data": {"id": 1, "project": {"id": 1, "name": "P"}} + "data": {"id": 1, "project": {"id": 1, "name": "P"}}, } mock_parse.return_value = { "event_type": "epic", "epic_id": None, - "assigned_by": "u" + "assigned_by": "u", } mock_coll.return_value = MagicMock() @@ -247,7 +298,7 @@ def test_epic_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, mock_n "/webhook/taiga?prj=P", data=json.dumps(payload), content_type="application/json", - headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"} + headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"}, ) assert resp.status_code == 400 @@ -255,17 +306,19 @@ def test_epic_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, mock_n @patch("routes.taiga_routes.get_collection") @patch("routes.taiga_routes.parse_taiga_event") @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) - def test_issue_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, mock_notify, client): + def test_issue_no_id_returns_400( + self, mock_verify, mock_parse, mock_coll, mock_notify, client + ): payload = { "type": "issue", "action": "create", "by": {"username": "u"}, - "data": {"id": 1, "project": {"id": 1, "name": "P"}} + "data": {"id": 1, "project": {"id": 1, "name": "P"}}, } mock_parse.return_value = { "event_type": "issue", "issue_id": None, - "assigned_by": "u" + "assigned_by": "u", } mock_coll.return_value = MagicMock() @@ -273,7 +326,7 @@ def test_issue_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, mock_ "/webhook/taiga?prj=P", data=json.dumps(payload), content_type="application/json", - headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"} + headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"}, ) assert resp.status_code == 400 @@ -281,8 +334,20 @@ def test_issue_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, mock_ @patch("routes.taiga_routes.get_collection") @patch("routes.taiga_routes.parse_taiga_event") @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) - def test_collection_name_task(self, mock_verify, mock_parse, mock_coll, mock_notify, client, taiga_task_payload): - mock_parse.return_value = {"event_type": "task", "task_id": 1, "assigned_by": "u"} + def test_collection_name_task( + self, + mock_verify, + mock_parse, + mock_coll, + mock_notify, + client, + taiga_task_payload, + ): + mock_parse.return_value = { + "event_type": "task", + "task_id": 1, + "assigned_by": "u", + } mock_coll.return_value = MagicMock() self._post(client, taiga_task_payload, prj="MyPrj") @@ -292,8 +357,20 @@ def test_collection_name_task(self, mock_verify, mock_parse, mock_coll, mock_not @patch("routes.taiga_routes.get_collection") @patch("routes.taiga_routes.parse_taiga_event") @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) - def test_collection_name_userstory(self, mock_verify, mock_parse, mock_coll, mock_notify, client, taiga_userstory_payload): - mock_parse.return_value = {"event_type": "userstory", "userstory_id": 1, "assigned_by": "u"} + def test_collection_name_userstory( + self, + mock_verify, + mock_parse, + mock_coll, + mock_notify, + client, + taiga_userstory_payload, + ): + mock_parse.return_value = { + "event_type": "userstory", + "userstory_id": 1, + "assigned_by": "u", + } mock_coll.return_value = MagicMock() self._post(client, taiga_userstory_payload, prj="MyPrj") diff --git a/tests/test_verify_signature_github.py b/tests/test_verify_signature_github.py index 75ce5ac..8c748e9 100644 --- a/tests/test_verify_signature_github.py +++ b/tests/test_verify_signature_github.py @@ -1,4 +1,5 @@ """Tests for routes/verify_signature/verify_signature_github.py""" + import hashlib, hmac, pytest from unittest.mock import MagicMock from routes.verify_signature.verify_signature_github import verify_github_signature diff --git a/tests/test_verify_signature_taiga.py b/tests/test_verify_signature_taiga.py index dbd1cb6..4affd32 100644 --- a/tests/test_verify_signature_taiga.py +++ b/tests/test_verify_signature_taiga.py @@ -1,4 +1,5 @@ """Tests for routes/verify_signature/verify_signature_taiga.py""" + import hashlib, hmac, pytest from unittest.mock import MagicMock from routes.verify_signature.verify_signature_taiga import verify_taiga_signature diff --git a/utils/datetime_utils.py b/utils/datetime_utils.py index db536c7..7edd72b 100644 --- a/utils/datetime_utils.py +++ b/utils/datetime_utils.py @@ -6,7 +6,7 @@ def to_madrid_local(ts: str) -> str: """ Receive date in ISO-8601 then transforms it to Europe/Madrid date. """ - if not ts: # '', None… + if not ts: # '', None… return ts # The date standard only accepts '+00:00', but taiga returns in 'Z' format dt_utc = datetime.fromisoformat(ts.replace("Z", "+00:00")) diff --git a/utils/recovery/github_recovery.py b/utils/recovery/github_recovery.py index ad29985..c67a3ae 100644 --- a/utils/recovery/github_recovery.py +++ b/utils/recovery/github_recovery.py @@ -14,10 +14,11 @@ setup_logging() logger = logging.getLogger(__name__) + def parse_dt(s: str | None) -> Optional[datetime]: - ''' + """ Accepts ‘2025-05-01’, ‘2025-05-01T14:30’, etc. And returns the datetime tz-aware (Madrid). - ''' + """ if not s: return None d = dtp.isoparse(s) @@ -35,9 +36,9 @@ def get_organization_repos(org: str, headers: Dict[str, str]) -> List[str]: def gh_paginated(url: str, headers: Dict[str, str]) -> Iterable[Dict]: - ''' + """ Gets paginated results from a GitHub API endpoint. With this each call to the API returns a suitable JSON - ''' + """ while url: r = requests.get(url, headers=headers, timeout=30) r.raise_for_status() @@ -46,85 +47,124 @@ def gh_paginated(url: str, headers: Dict[str, str]) -> Iterable[Dict]: def upsert(coll, docs: list[dict], key: str) -> int: - ''' - Upserts a list of documents into a MongoDB collection. - ''' + """ + Upserts a list of documents into a MongoDB collection. + """ if not docs: return 0 - operations = [UpdateOne({key: d[key]}, {"$set": d}, upsert=True) for d in docs] #Create a list of UpdateOne operations for each document in the list, using the key to identify the document + operations = [ + UpdateOne({key: d[key]}, {"$set": d}, upsert=True) for d in docs + ] # Create a list of UpdateOne operations for each document in the list, using the key to identify the document res = coll.bulk_write(operations, ordered=False) return res.matched_count + len(res.upserted_ids) - - -def collect_github(org: str, repo: str, prj: str, events: list[str], since: Optional[str], until: Optional[str], quality_model: str): - ''' +def collect_github( + org: str, + repo: str, + prj: str, + events: list[str], + since: Optional[str], + until: Optional[str], + quality_model: str, +): + """ Collects data from a GitHub repository and inserts it into MongoDB. Follows the schema of the LD-Connect project. - ''' - repo_full = f"{org}/{repo}" # Full name of the repository, e.g. "LD-Connect/ld-connect" + """ + repo_full = ( + f"{org}/{repo}" # Full name of the repository, e.g. "LD-Connect/ld-connect" + ) headers = {"Accept": "application/vnd.github+json"} if GITHUB_TOKEN: - headers["Authorization"] = f"Bearer {GITHUB_TOKEN}" # Authentication with GitHub API using a token - - counters = {"commits": 0, "issues": 0, "pull_requests": 0} #Counter to display the number of documents inserted of each event type - author_login = "backfill" # The author login is always "backfill" for backfilling - - for ev in events: # Start iterating over the events to collect - - if ev == "commits": #First commits - event_name= "push" # The event name for commits is always "push" - + headers["Authorization"] = ( + f"Bearer {GITHUB_TOKEN}" # Authentication with GitHub API using a token + ) + + counters = { + "commits": 0, + "issues": 0, + "pull_requests": 0, + } # Counter to display the number of documents inserted of each event type + author_login = "backfill" # The author login is always "backfill" for backfilling + + for ev in events: # Start iterating over the events to collect + + if ev == "commits": # First commits + event_name = "push" # The event name for commits is always "push" + log_url = f"https://api.github.com/repos/{repo_full}/commits?per_page=100" - if since: log_url += f"&since={since}" # If a SINCE date is proviaded, add it to the URL - if until: log_url += f"&until={until}" # If a UNTIL date is proviaded, add it to the URL + if since: + log_url += ( + f"&since={since}" # If a SINCE date is proviaded, add it to the URL + ) + if until: + log_url += ( + f"&until={until}" # If a UNTIL date is proviaded, add it to the URL + ) - payloads = [] #List to store the payloads of the commits - for c in gh_paginated(log_url, headers): # Iterate over the paginated results of the commits and store them in the payloads list under the schema - payloads.append({ - "X-GitHub-Event": "push", - "repository": {"full_name": repo_full}, - "organization": {"login": org}, - "sender": c["author"] or {}, - "commits": [{ - "id": c["sha"], - "url": c["url"], - "message": c["commit"]["message"], - "timestamp": c["commit"]["author"]["date"], - "author": { - "username": (c["author"] or {}).get("login", ""), - "name": c["commit"]["author"]["name"], - "email": c["commit"]["author"]["email"], - }, - }], - }) - - coll = get_collection(f"github_{prj}.commits") # Collection name to store - for raw in payloads: # All the raw payloads, parse them and insert in collection - doc = parse_github_event(raw,prj) + payloads = [] # List to store the payloads of the commits + for c in gh_paginated( + log_url, headers + ): # Iterate over the paginated results of the commits and store them in the payloads list under the schema + payloads.append( + { + "X-GitHub-Event": "push", + "repository": {"full_name": repo_full}, + "organization": {"login": org}, + "sender": c["author"] or {}, + "commits": [ + { + "id": c["sha"], + "url": c["url"], + "message": c["commit"]["message"], + "timestamp": c["commit"]["author"]["date"], + "author": { + "username": (c["author"] or {}).get("login", ""), + "name": c["commit"]["author"]["name"], + "email": c["commit"]["author"]["email"], + }, + } + ], + } + ) + + coll = get_collection(f"github_{prj}.commits") # Collection name to store + for ( + raw + ) in payloads: # All the raw payloads, parse them and insert in collection + doc = parse_github_event(raw, prj) for c in doc["commits"]: - c.update({"team_name": doc["team_name"], - "repo_name": doc["repo_name"], - "sender_info": doc["sender_info"], - "prj": prj}) + c.update( + { + "team_name": doc["team_name"], + "repo_name": doc["repo_name"], + "sender_info": doc["sender_info"], + "prj": prj, + } + ) counters["commits"] += upsert(coll, [c], "sha") - #COMMUNICATION WITH LD_EVAL USING API - logger.info(f"Notifying LD_EVAL about event: {event_name} for team with external_id: {prj} with quality_model: {quality_model}") + # COMMUNICATION WITH LD_EVAL USING API + logger.info( + f"Notifying LD_EVAL about event: {event_name} for team with external_id: {prj} with quality_model: {quality_model}" + ) try: notify_eval_push(event_name, prj, author_login, quality_model) except Exception as e: logger.error(f"Error notifying LD_EVAL: {e}") - - - - elif ev == "issues": # Issue event - event_name = "issues" # The event name for issues is always "issues" + + elif ev == "issues": # Issue event + event_name = "issues" # The event name for issues is always "issues" url = f"https://api.github.com/repos/{repo_full}/issues?state=all&per_page=100" - if since: url += f"&since={since}" + if since: + url += f"&since={since}" raw_issues = list(gh_paginated(url, headers)) - coll = get_collection(f"github_{prj}.issues") #Collection name to store - for i in raw_issues: # All the raw payloads, parse them and insert in collection + coll = get_collection(f"github_{prj}.issues") # Collection name to store + for ( + i + ) in ( + raw_issues + ): # All the raw payloads, parse them and insert in collection payload = { "X-GitHub-Event": "issues", "action": i["state"], @@ -137,19 +177,22 @@ def collect_github(org: str, repo: str, prj: str, events: list[str], since: Opt doc["prj"] = prj doc["issue_id"] = doc["issue"]["number"] counters["issues"] += upsert(coll, [doc], "issue_id") - #COMMUNICATION WITH LD_EVAL USING API - logger.info(f"Notifying LD_EVAL about event: {event_name} for team with external_id: {prj} with quality_model: {quality_model}") + # COMMUNICATION WITH LD_EVAL USING API + logger.info( + f"Notifying LD_EVAL about event: {event_name} for team with external_id: {prj} with quality_model: {quality_model}" + ) try: notify_eval_push(event_name, prj, author_login, quality_model) except Exception as e: logger.error(f"Error notifying LD_EVAL: {e}") - - - elif ev == "pull_requests": #Pull request event - event_name = "pull_requests" # The event name for pull requests is always "pull_requests" + + elif ev == "pull_requests": # Pull request event + event_name = "pull_requests" # The event name for pull requests is always "pull_requests" url = f"https://api.github.com/repos/{repo_full}/pulls?state=closed&per_page=100" raw_prs = list(gh_paginated(url, headers)) - coll = get_collection(f"github_{prj}.pull_requests") # All the raw payloads, parse them and insert in collection + coll = get_collection( + f"github_{prj}.pull_requests" + ) # All the raw payloads, parse them and insert in collection for p in raw_prs: payload = { "X-GitHub-Event": "pull_request", @@ -162,56 +205,105 @@ def collect_github(org: str, repo: str, prj: str, events: list[str], since: Opt doc = parse_github_event(payload, prj) doc["prj"] = prj counters["pull_requests"] += upsert(coll, [doc], "pr_number") - #COMMUNICATION WITH LD_EVAL USING API - logger.info(f"Notifying LD_EVAL about event: {event_name} for team with external_id: {prj} with quality_model: {quality_model}") + # COMMUNICATION WITH LD_EVAL USING API + logger.info( + f"Notifying LD_EVAL about event: {event_name} for team with external_id: {prj} with quality_model: {quality_model}" + ) try: notify_eval_push(event_name, prj, author_login, quality_model) except Exception as e: logger.error(f"Error notifying LD_EVAL: {e}") - - + else: logger.warning("Event %s not supported.", ev) - for k in ("commits", "issues", "pull_requests"): # Log the counters of each event type + for k in ( + "commits", + "issues", + "pull_requests", + ): # Log the counters of each event type if k in events: - logger.info(" • %s → %d documents", k.replace('_', ' '), counters[k]) + logger.info(" • %s → %d documents", k.replace("_", " "), counters[k]) logger.info("%d documents inserted.", sum(counters.values())) - - if __name__ == "__main__": - ap = argparse.ArgumentParser(description="Back-fill of GITHUB for a project, to insert it in MongoDB") - ap.add_argument("--org", required=True, help="Organization name on Github") - ap.add_argument("--repo", help="Repository name on Github, if not provided, it will backfill all repositories in the organization") - ap.add_argument("--prj", required=True, help="External ID of the project in LD-Connect") - ap.add_argument("--events", default="commits,issues,pull_requests",help="List separated by commas pf the events to backfill, by default: commits,issues,pull_requests") - ap.add_argument("--from-date", help="Date FROM the backfill will be made, must be in format (YYYY-MM-DD), can also put a full date with time (2025-05-01T14:30)") - ap.add_argument("--to-date", help="Date UNTIL the backfill will be made, must be in format (YYYY-MM-DD), can also put a full date with time (2025-05-01T14:30)") - ap.add_argument("--quality-model", default="default", help="Sets the quality model to use for the evaluation, by default: default") + ap = argparse.ArgumentParser( + description="Back-fill of GITHUB for a project, to insert it in MongoDB" + ) + ap.add_argument("--org", required=True, help="Organization name on Github") + ap.add_argument( + "--repo", + help="Repository name on Github, if not provided, it will backfill all repositories in the organization", + ) + ap.add_argument( + "--prj", required=True, help="External ID of the project in LD-Connect" + ) + ap.add_argument( + "--events", + default="commits,issues,pull_requests", + help="List separated by commas pf the events to backfill, by default: commits,issues,pull_requests", + ) + ap.add_argument( + "--from-date", + help="Date FROM the backfill will be made, must be in format (YYYY-MM-DD), can also put a full date with time (2025-05-01T14:30)", + ) + ap.add_argument( + "--to-date", + help="Date UNTIL the backfill will be made, must be in format (YYYY-MM-DD), can also put a full date with time (2025-05-01T14:30)", + ) + ap.add_argument( + "--quality-model", + default="default", + help="Sets the quality model to use for the evaluation, by default: default", + ) ns = ap.parse_args() events = [e.strip() for e in ns.events.split(",") if e.strip()] - since = parse_dt(ns.from_date).astimezone(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") if ns.from_date else None - until = parse_dt(ns.to_date).astimezone(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") if ns.to_date else None + since = ( + parse_dt(ns.from_date) + .astimezone(timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z") + if ns.from_date + else None + ) + until = ( + parse_dt(ns.to_date) + .astimezone(timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z") + if ns.to_date + else None + ) headers = {"Accept": "application/vnd.github+json"} if GITHUB_TOKEN: - headers["Authorization"] = f"Bearer {GITHUB_TOKEN}" # Authentication with GitHub API using a token + headers["Authorization"] = ( + f"Bearer {GITHUB_TOKEN}" # Authentication with GitHub API using a token + ) if ns.repo: - repos= [ns.repo] + repos = [ns.repo] else: repos = get_organization_repos(ns.org, headers) - + if not repos: raise SystemExit(f"No repositories found for organization {ns.org}.") - - + for repo in repos: - collect_github( org = ns.org, repo = repo, prj= ns.prj, events= events, since = since, until= until, quality_model=ns.quality_model) + collect_github( + org=ns.org, + repo=repo, + prj=ns.prj, + events=events, + since=since, + until=until, + quality_model=ns.quality_model, + ) # In order to execute this script: python -m recovery.github_recovery --org LD-Connect --repo ld-connect --prj LD_Test_Project --events commits,issues,pull_requests --from-date 2025-01-01 --to-date 2025-12-31 -# Or python -m recovery.github_recovery --org LD-Connect --repo ld-connect --prj LD_Test_Project \ No newline at end of file +# Or python -m recovery.github_recovery --org LD-Connect --repo ld-connect --prj LD_Test_Project diff --git a/utils/recovery/taiga_recovery.py b/utils/recovery/taiga_recovery.py index ad51db1..f50211f 100644 --- a/utils/recovery/taiga_recovery.py +++ b/utils/recovery/taiga_recovery.py @@ -2,7 +2,7 @@ from pymongo import UpdateOne from datetime import datetime, timezone from typing import Optional, Dict, List -from dateutil import tz, parser as dtparser # pip install python-dateutil +from dateutil import tz, parser as dtparser # pip install python-dateutil import logging from database.mongo_client import get_collection @@ -16,7 +16,6 @@ logger = logging.getLogger(__name__) - def parse_dt(s: str) -> datetime: """Accepts ‘2025-05-01’, ‘2025-05-01T14:30’, etc. And returns the datetime tz-aware (Madrid).""" d = dtparser.isoparse(s) @@ -27,10 +26,10 @@ def parse_dt(s: str) -> datetime: # Functions to interact with Taiga API def get_username_id(token: str) -> int: - ''' + """ Given a Taiga API token, return the user ID of the authenticated user. With this ID we canfind the projects that the user is a member of. - ''' + """ h = {"Authorization": f"Bearer {token}"} r = requests.get(f"https://api.taiga.io/api/v1/users/me", headers=h, timeout=10) r.raise_for_status() @@ -44,20 +43,28 @@ def get_project_id_by_slug(slug: str) -> int: if r.status_code == 200: return r.json()["id"] if r.status_code in (401, 403): - raise SystemExit(f"Project ‘{slug}’ is private. Provide credentials or make it public.") + raise SystemExit( + f"Project ‘{slug}’ is private. Provide credentials or make it public." + ) if r.status_code == 404: raise SystemExit(f"Project slug ‘{slug}’ not found.") r.raise_for_status() assert False # pragma: no cover - + + def get_project_id_by_username_id(project_name: str, token: str) -> int: - ''' + """ Given a project name and a Taiga API token, return the project ID. With the projecta ID we can fetch the entities (tasks, issues, etc.) of the project. - ''' + """ h = {"Authorization": f"Bearer {token}"} uid = get_username_id(token) - r = requests.get(f"https://api.taiga.io/api/v1/projects", headers=h, params={"member": uid}, timeout=10) + r = requests.get( + f"https://api.taiga.io/api/v1/projects", + headers=h, + params={"member": uid}, + timeout=10, + ) r.raise_for_status() for p in r.json(): if p["name"].lower() == project_name.lower(): @@ -65,129 +72,152 @@ def get_project_id_by_username_id(project_name: str, token: str) -> int: raise SystemExit(f"Project named «{project_name}» is not under the introduced user") -def fetch_entities(entity: str, project: int, start: Optional[datetime] = None, end: Optional[datetime] = None) -> List[Dict]: - ''' +def fetch_entities( + entity: str, + project: int, + start: Optional[datetime] = None, + end: Optional[datetime] = None, +) -> List[Dict]: + """ Given an entity type (tasks, issues, epics, userstories), a project ID, and a token, the function fetches the entities from the Taiga API. The start and end parameters are optional and can be used to filter the entities by creation date. - - ''' + + """ if entity not in ENTITY_ENDPOINT: raise ValueError(f"Not supported type: {entity}") - endpoint_path = ENTITY_ENDPOINT[entity][0] + endpoint_path = ENTITY_ENDPOINT[entity][0] headers = { "x-disable-pagination": "True", } params = {"project": project} - if start: params["modified_date__gte"] = start.astimezone(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") # See if there is a start date to fetch data, if there is add it to the params - if end: params["modified_date__lte"] = end.astimezone(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") # See if there is a end date to fetch data, if there is add it to the params - - r = requests.get(f"https://api.taiga.io/api/v1/{endpoint_path}", headers=headers, params=params, timeout=30) # In the request we add the headers and params + if start: + params["modified_date__gte"] = ( + start.astimezone(timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z") + ) # See if there is a start date to fetch data, if there is add it to the params + if end: + params["modified_date__lte"] = ( + end.astimezone(timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z") + ) # See if there is a end date to fetch data, if there is add it to the params + + r = requests.get( + f"https://api.taiga.io/api/v1/{endpoint_path}", + headers=headers, + params=params, + timeout=30, + ) # In the request we add the headers and params r.raise_for_status() return r.json() - def upsert(coll, docs: list[dict], key: str) -> int: - ''' - Upserts a list of documents into a MongoDB collection. - ''' + """ + Upserts a list of documents into a MongoDB collection. + """ if not docs: return 0 - operations = [UpdateOne({key: d[key]}, {"$set": d}, upsert=True) for d in docs] #Create a list of UpdateOne operations for each document in the list, using the key to identify the document + operations = [ + UpdateOne({key: d[key]}, {"$set": d}, upsert=True) for d in docs + ] # Create a list of UpdateOne operations for each document in the list, using the key to identify the document res = coll.bulk_write(operations, ordered=False) return res.matched_count + len(res.upserted_ids) # Converters from API Schema to MongoDB Schema def task_from_api(j: dict, prj: str) -> dict: - ''' + """ Converts a task JSON object from the Taiga API to a MongoDB document schema. - ''' + """ m = j.get("milestone_extra_info") or {} us = j.get("user_story_extra_info") or {} return { - "task_id": j["id"], - "action_type": "import", - "assigned_by": "backfill", - "assigned_to": (j.get("assigned_to_extra_info") or {}).get("username"), - "created_date": j["created_date"], + "task_id": j["id"], + "action_type": "import", + "assigned_by": "backfill", + "assigned_to": (j.get("assigned_to_extra_info") or {}).get("username"), + "created_date": j["created_date"], "custom_attributes": j.get("custom_attributes_values") or {}, "estimated_finish": m.get("estimated_finish"), - "estimated_start": m.get("estimated_start"), - "event_type": "task", - "finished_date": j["finished_date"], - "is_closed": j["status_extra_info"]["is_closed"], + "estimated_start": m.get("estimated_start"), + "event_type": "task", + "finished_date": j["finished_date"], + "is_closed": j["status_extra_info"]["is_closed"], "milestone_closed": m.get("closed"), "milestone_created_date": m.get("created_date"), - "milestone_id": j.get("milestone"), + "milestone_id": j.get("milestone"), "milestone_modified_date": m.get("modified_date"), "milestone_name": m.get("name"), - "modified_date": j["modified_date"], - "prj": prj, - "reference": j["ref"], - "status": j["status_extra_info"]["name"], - "subject": j["subject"], - "team_name": j["project_extra_info"]["name"], - "userstory_id": j.get("user_story"), - "userstory_is_closed": us.get("is_closed"), + "modified_date": j["modified_date"], + "prj": prj, + "reference": j["ref"], + "status": j["status_extra_info"]["name"], + "subject": j["subject"], + "team_name": j["project_extra_info"]["name"], + "userstory_id": j.get("user_story"), + "userstory_is_closed": us.get("is_closed"), } def issue_from_api(j: dict, prj: str) -> dict: - ''' + """ Converts a isue JSON object from the Taiga API to a MongoDB document schema. - ''' + """ return { - "issue_id": j["id"], + "issue_id": j["id"], "action_type": "import", - "assigned_by": "backfill", + "assigned_by": "backfill", "assigned_to": (j.get("assigned_to_extra_info") or {}).get("username"), - "created_date": j["created_date"], + "created_date": j["created_date"], "description": j.get("description"), - "due_date": j.get("due_date"), - "event_type": "issue", + "due_date": j.get("due_date"), + "event_type": "issue", "finished_date": j.get("finished_date"), - "is_closed": (j.get("status_extra_info") or {}).get("is_closed"), + "is_closed": (j.get("status_extra_info") or {}).get("is_closed"), "modified_date": j["modified_date"], - "priority": (j.get("priority_extra_info") or {}).get("name"), - "prj": prj, - "severity": (j.get("severity_extra_info") or {}).get("name"), - "status": (j.get("status_extra_info") or {}).get("name"), - "subject": j["subject"], - "team_name": j["project_extra_info"]["name"], - "type": (j.get("type_extra_info") or {}).get("name"), + "priority": (j.get("priority_extra_info") or {}).get("name"), + "prj": prj, + "severity": (j.get("severity_extra_info") or {}).get("name"), + "status": (j.get("status_extra_info") or {}).get("name"), + "subject": j["subject"], + "team_name": j["project_extra_info"]["name"], + "type": (j.get("type_extra_info") or {}).get("name"), } def epic_from_api(j: dict, prj: str) -> dict: - ''' + """ Converts an epic JSON object from the Taiga API to a MongoDB document schema. - ''' + """ return { - "epic_id": j["id"], - "action_type": "import", - "assigned_by": "backfill", - "created_date": j["created_date"], - "event_type": "epic", - "is_closed": (j.get("status_extra_info") or {}).get("is_closed"), + "epic_id": j["id"], + "action_type": "import", + "assigned_by": "backfill", + "created_date": j["created_date"], + "event_type": "epic", + "is_closed": (j.get("status_extra_info") or {}).get("is_closed"), "modified_date": j["modified_date"], - "prj": prj, - "project_id": j["project_extra_info"]["id"], - "status": (j.get("status_extra_info") or {}).get("name"), - "subject": j["subject"], - "team_name": j["project_extra_info"]["name"], + "prj": prj, + "project_id": j["project_extra_info"]["id"], + "status": (j.get("status_extra_info") or {}).get("name"), + "subject": j["subject"], + "team_name": j["project_extra_info"]["name"], } - - + + def userstory_from_api(j: dict, prj: str) -> dict: - ''' + """ Converts an userstory JSON object from the Taiga API to a MongoDB document schema. - ''' + """ m = j.get("milestone_extra_info") or {} desc = j.get("description") or "" pattern = bool(re.search(r"as\s+.*?\s+i want\s+.*?\s+so that\s+.*", desc, re.I)) - raw_points = j.get("points") # puede ser list | "" | None + raw_points = j.get("points") # puede ser list | "" | None if isinstance(raw_points, list): total = sum((p.get("value") or 0) for p in raw_points) else: @@ -196,58 +226,78 @@ def userstory_from_api(j: dict, prj: str) -> dict: return { "userstory_id": j["id"], "action_type": "import", - "assigned_by": "backfill", - "created_date": j["created_date"], + "assigned_by": "backfill", + "created_date": j["created_date"], "custom_attributes": j.get("custom_attributes_values") or {}, "estimated_finish": m.get("estimated_finish"), - "estimated_start": m.get("estimated_start"), - "event_type": "userstory", - "is_closed": (j.get("status_extra_info") or {}).get("is_closed"), + "estimated_start": m.get("estimated_start"), + "event_type": "userstory", + "is_closed": (j.get("status_extra_info") or {}).get("is_closed"), "milestone_closed": m.get("closed"), "milestone_created_date": m.get("created_date"), - "milestone_id": j.get("milestone"), + "milestone_id": j.get("milestone"), "milestone_modified_date": m.get("modified_date"), "milestone_name": m.get("name"), "modified_date": j["modified_date"], "pattern": pattern, - "priority": (j.get("custom_attributes_values") or {}).get("Priority"), - "prj": prj, - "status": (j.get("status_extra_info") or {}).get("name"), - "subject": j["subject"], - "team_name": j["project_extra_info"]["name"], - "total_points": total, + "priority": (j.get("custom_attributes_values") or {}).get("Priority"), + "prj": prj, + "status": (j.get("status_extra_info") or {}).get("name"), + "subject": j["subject"], + "team_name": j["project_extra_info"]["name"], + "total_points": total, } - - -ENTITY_ENDPOINT = { - "task": ("tasks", task_from_api, "task_id"), - "issue": ("issues", issue_from_api, "issue_id"), - "epic": ("epics", epic_from_api, "epic_id"), - "userstory": ("userstories", userstory_from_api, "userstory_id"), - } - +ENTITY_ENDPOINT = { + "task": ("tasks", task_from_api, "task_id"), + "issue": ("issues", issue_from_api, "issue_id"), + "epic": ("epics", epic_from_api, "epic_id"), + "userstory": ("userstories", userstory_from_api, "userstory_id"), +} def main(argv: list[str] | None = None): - ap = argparse.ArgumentParser(description="Back-fill of Taiga for a project, to insert it in MongoDB") - #ap.add_argument("--project", required=True, help="Name of the project in Taiga") - ap.add_argument("--slug", "--project", dest="slug", required=True,help="Slug públic del projecte a Taiga") - #ap.add_argument("--project", required=True, help="Name of the project in Taiga") - ap.add_argument("--prj", required=True, help="External ID of the project in LD-Connect") + ap = argparse.ArgumentParser( + description="Back-fill of Taiga for a project, to insert it in MongoDB" + ) + # ap.add_argument("--project", required=True, help="Name of the project in Taiga") + ap.add_argument( + "--slug", + "--project", + dest="slug", + required=True, + help="Slug públic del projecte a Taiga", + ) + # ap.add_argument("--project", required=True, help="Name of the project in Taiga") + ap.add_argument( + "--prj", required=True, help="External ID of the project in LD-Connect" + ) # ap.add_argument("--taiga-user", required=True, help="Username in Taiga of the teacher with acces to all Students Projects") # ap.add_argument("--taiga-pass", required=True, help="Password in Taiga of the teacher with acces to all Students Projects") - ap.add_argument("--events", default="task,userstory,issue,epic",help="List separated by commas pf the events to backfill, by default: task,userstory,issue,epic") - ap.add_argument("--from-date", help="Date FROM the backfill will be made, must be in format (2025-05-01), can also put a full date with time (2025-05-01T14:30)") - ap.add_argument("--to-date", help="Date UNTIL the backfill will be made, must be in format (2025-05-01), can also put a full date with time (2025-05-01T14:30)") - ap.add_argument("--quality-model", default="default", help="Sets the quality model to use for the evaluation, by default: default") - - - ns = ap.parse_args(argv) - events = [e.strip().lower() for e in ns.events.split(",") if e.strip()] - start = parse_dt(ns.from_date) if ns.from_date else None - end = parse_dt(ns.to_date) if ns.to_date else None + ap.add_argument( + "--events", + default="task,userstory,issue,epic", + help="List separated by commas pf the events to backfill, by default: task,userstory,issue,epic", + ) + ap.add_argument( + "--from-date", + help="Date FROM the backfill will be made, must be in format (2025-05-01), can also put a full date with time (2025-05-01T14:30)", + ) + ap.add_argument( + "--to-date", + help="Date UNTIL the backfill will be made, must be in format (2025-05-01), can also put a full date with time (2025-05-01T14:30)", + ) + ap.add_argument( + "--quality-model", + default="default", + help="Sets the quality model to use for the evaluation, by default: default", + ) + + ns = ap.parse_args(argv) + events = [e.strip().lower() for e in ns.events.split(",") if e.strip()] + start = parse_dt(ns.from_date) if ns.from_date else None + end = parse_dt(ns.to_date) if ns.to_date else None # Payload with the credentials to get the token # payload = { @@ -255,39 +305,47 @@ def main(argv: list[str] | None = None): # "password": TAIGA_PASSWORD, # "type": "normal" # } - + # token = get_token(payload) #Get the token using the credentials provided in the payload # print(f"Using token: {token}") # Print the token to the console, this is for debugging purposes - pid = get_project_id_by_slug(ns.slug) # Get the project ID using the project name and the token info - + pid = get_project_id_by_slug( + ns.slug + ) # Get the project ID using the project name and the token info - total = 0 - for event in events: # Iterate over the events to backfill + total = 0 + for event in events: # Iterate over the events to backfill endpoint, converter, key = ENTITY_ENDPOINT[event] - raw = fetch_entities(event, pid, start, end) # Get the raw data from the Taiga API for the event - docs = [converter(r, ns.prj) for r in raw] # Convert the raw data to the MongoDB schema using the converter function - coll = get_collection(f"taiga_{ns.prj}.{event}") # Get the MongoDB collection for the event - n = upsert(coll, docs, key) # Upsert the documents + raw = fetch_entities( + event, pid, start, end + ) # Get the raw data from the Taiga API for the event + docs = [ + converter(r, ns.prj) for r in raw + ] # Convert the raw data to the MongoDB schema using the converter function + coll = get_collection( + f"taiga_{ns.prj}.{event}" + ) # Get the MongoDB collection for the event + n = upsert(coll, docs, key) # Upsert the documents total += n logger.info(" • %s → %d documents", event, n) - - #COMMUNICATION WITH LD_EVAL USING API - logger.info(f"Notifying LD_EVAL about event: {event} for team with external_id: {ns.prj} with quality_model: {ns.quality_model}") + + # COMMUNICATION WITH LD_EVAL USING API + logger.info( + f"Notifying LD_EVAL about event: {event} for team with external_id: {ns.prj} with quality_model: {ns.quality_model}" + ) try: notify_eval_push(event, ns.prj, "backfill", ns.quality_model) except Exception as e: logger.error(f"Error notifying LD_EVAL: {e}") - - - span = "all time" if not (start or end) else \ - f"from {ns.from_date or '…'} to {ns.to_date or '…'}" + span = ( + "all time" + if not (start or end) + else f"from {ns.from_date or '…'} to {ns.to_date or '…'}" + ) logger.info("%d documents inserted (%s)", total, span) - - if __name__ == "__main__": main() - -# In order to execute this script: python -m recovery.taiga_recovery --project LD_Test_Project --prj LD_Test_Project \ No newline at end of file + +# In order to execute this script: python -m recovery.taiga_recovery --project LD_Test_Project --prj LD_Test_Project diff --git a/utils/taiga_token/get_taiga_token.py b/utils/taiga_token/get_taiga_token.py index 43a36be..a702d32 100644 --- a/utils/taiga_token/get_taiga_token.py +++ b/utils/taiga_token/get_taiga_token.py @@ -4,30 +4,27 @@ logger = logging.getLogger(__name__) -def get_token(payload:dict) -> str: +def get_token(payload: dict) -> str: """ Using a POST request of the API, this function retrieves the authentication token from Taiga. The payload must contain the username and password of the user. - + """ # Define the login endpoint URL and payload login_url = "https://api.taiga.io/api/v1/auth" - # Send the POST request to log in response = requests.post(login_url, json=payload) response.raise_for_status() # Will raise an error if the response status is not 200 # Parse the JSON response data = response.json() - # Extract the token; + # Extract the token; token = data.get("auth_token") if token: logger.info("Login successful, token retrieved.") else: logger.error("Login failed or token not found in the response.") - - return token - + return token diff --git a/utils/taiga_token/taiga_auth.py b/utils/taiga_token/taiga_auth.py index 8f9b099..0488c04 100644 --- a/utils/taiga_token/taiga_auth.py +++ b/utils/taiga_token/taiga_auth.py @@ -1,29 +1,27 @@ import requests, logging, time log = logging.getLogger(__name__) -_TOKENS = {} # key = (username, password) -> token +_TOKENS = {} # key = (username, password) -> token -def get_taiga_token(username:str, password: str) -> str: - ''' + +def get_taiga_token(username: str, password: str) -> str: + """ Tool to get a Taiga API token and cache it for 23 hours (In taiga documentation, says the token expires in 24h). If the token is about to expire (less than 60 seconds left), it will request a new one. This avoids making too many requests to the Taiga API for the token. - ''' - - + """ + key = (username, password) token, exp = _TOKENS.get(key, (None, 0)) - + # If the token is not set or is about to expire, request a new one if token is None or exp - time.time() < 60: - payload = { - "username": username, - "password": password, - "type": "normal" - } - r = requests.post("https://api.taiga.io/api/v1/auth", json=payload, timeout=(2, 5)) + payload = {"username": username, "password": password, "type": "normal"} + r = requests.post( + "https://api.taiga.io/api/v1/auth", json=payload, timeout=(2, 5) + ) r.raise_for_status() - + token = r.json()["auth_token"] exp = time.time() + 23 * 3600 _TOKENS[key] = (token, exp) diff --git a/utils/webhook_deletion/central_webhook_deletion.py b/utils/webhook_deletion/central_webhook_deletion.py index 8dc1607..292ff05 100644 --- a/utils/webhook_deletion/central_webhook_deletion.py +++ b/utils/webhook_deletion/central_webhook_deletion.py @@ -4,6 +4,7 @@ from config.settings import TAIGA_USERNAME, TAIGA_PASSWORD from dotenv import load_dotenv import os + # Load environment variables from the .env file load_dotenv() @@ -14,16 +15,16 @@ payload = { "username": TAIGA_USERNAME, "password": TAIGA_PASSWORD, - "type": "normal", + "type": "normal", } if __name__ == "__main__": # Get the token from Taiga token = get_token(payload) - + # Delete webhooks from GitHub delete_all_github_webhooks(WEBHOOK_URL_GITHUB) - + # Delete webhooks from Taiga - delete_all_taiga_webhooks(token, WEBHOOK_URL_TAIGA) \ No newline at end of file + delete_all_taiga_webhooks(token, WEBHOOK_URL_TAIGA) diff --git a/utils/webhook_deletion/delete_webhooks_github.py b/utils/webhook_deletion/delete_webhooks_github.py index 609923a..a87deec 100644 --- a/utils/webhook_deletion/delete_webhooks_github.py +++ b/utils/webhook_deletion/delete_webhooks_github.py @@ -1,22 +1,27 @@ import logging import pymongo import requests -from config.settings import MONGO_URI, MONGO_DB, GITHUB_TOKEN, WEBHOOK_URL_GITHUB, GITHUB_API_URL +from config.settings import ( + MONGO_URI, + MONGO_DB, + GITHUB_TOKEN, + WEBHOOK_URL_GITHUB, + GITHUB_API_URL, +) logger = logging.getLogger(__name__) - -def list_github_hooks(owner, repo)-> list: +def list_github_hooks(owner, repo) -> list: """ - The function lists all the webhooks created on a specific repository, we need the owner and the repository name. + The function lists all the webhooks created on a specific repository, we need the owner and the repository name. Also a owner or admin token is needed to authenticate the request. """ url = f"{GITHUB_API_URL}/repos/{owner}/{repo}/hooks" headers = { "Accept": "application/vnd.github.v3+json", - "Authorization": f"Bearer {GITHUB_TOKEN}" + "Authorization": f"Bearer {GITHUB_TOKEN}", } resp = requests.get(url, headers=headers) resp.raise_for_status() @@ -24,55 +29,58 @@ def list_github_hooks(owner, repo)-> list: def delete_github_hook(owner, repo, hook_id): - ''' - This function deletes a specific webhook, we need the owner, the repository name and the id of the webhook to delete. + """ + This function deletes a specific webhook, we need the owner, the repository name and the id of the webhook to delete. Also a owner or admin token is needed to authenticate the request. - ''' + """ url = f"{GITHUB_API_URL}/repos/{owner}/{repo}/hooks/{hook_id}" headers = { "Accept": "application/vnd.github.v3+json", - "Authorization": f"Bearer {GITHUB_TOKEN}" + "Authorization": f"Bearer {GITHUB_TOKEN}", } resp = requests.delete(url, headers=headers) resp.raise_for_status() return resp - def delete_all_github_webhooks(webhook_url_github): - ''' + """ This function deletes all the webhooks created on the repositories of the database. - ''' - mongo_client = pymongo.MongoClient(MONGO_URI) # URI to connect to the database - db = mongo_client[MONGO_DB] # Name of the database + """ + mongo_client = pymongo.MongoClient(MONGO_URI) # URI to connect to the database + db = mongo_client[MONGO_DB] # Name of the database # We get all collections with 'commits' in their name and store them in a list. From them we will extract the repositories names and owners. all_collections = db.list_collection_names() github_commit_collections = [c for c in all_collections if "commit" in c] - - - #We iterate over the collections and the repositories to delete the webhooks - #We first get the repositories in the collection and then we get the hooks for each repository + # We iterate over the collections and the repositories to delete the webhooks + # We first get the repositories in the collection and then we get the hooks for each repository for col_name in github_commit_collections: distinct_repos = db[col_name].distinct("repository") - + for repo_full_name in distinct_repos: - owner, repo = repo_full_name.split('/') + owner, repo = repo_full_name.split("/") try: hooks = list_github_hooks(owner, repo) logger.info("Found %d hooks for %s/%s", len(hooks), owner, repo) except requests.HTTPError as e: logger.error("Error listing hooks for %s/%s: %s", owner, repo, e) continue - + for hook in hooks: config_url = hook.get("config", {}).get("url", "") hook_id = hook.get("id") - + if config_url == webhook_url_github: # Delete it try: delete_github_hook(owner, repo, hook_id) logger.info("Deleted hook %s for %s/%s", hook_id, owner, repo) except requests.HTTPError as e: - logger.error("Error deleting hook %s for %s/%s: %s", hook_id, owner, repo, e) \ No newline at end of file + logger.error( + "Error deleting hook %s for %s/%s: %s", + hook_id, + owner, + repo, + e, + ) diff --git a/utils/webhook_deletion/delete_webhooks_taiga.py b/utils/webhook_deletion/delete_webhooks_taiga.py index d20b23d..44323af 100644 --- a/utils/webhook_deletion/delete_webhooks_taiga.py +++ b/utils/webhook_deletion/delete_webhooks_taiga.py @@ -6,49 +6,41 @@ logger = logging.getLogger(__name__) - -#We define two functions to list and delete the webhooks using the API of github. -#The first function lists all the webhooks created on a repository, we need the owner and the repository name. Also a owner or admin token is needed to authenticate the request. +# We define two functions to list and delete the webhooks using the API of github. +# The first function lists all the webhooks created on a repository, we need the owner and the repository name. Also a owner or admin token is needed to authenticate the request. def list_taiga_hooks(project_id, token): url = f"https://api.taiga.io/api/v1/webhooks?project={project_id}" - headers = { - "Authorization": f"Bearer {token}" - } + headers = {"Authorization": f"Bearer {token}"} resp = requests.get(url, headers=headers) resp.raise_for_status() return resp.json() -#The second function deletes a webhook, we need the owner, the repository name and the id of the webhook to delete. + +# The second function deletes a webhook, we need the owner, the repository name and the id of the webhook to delete. # Also a owner or admin token is needed to authenticate the request. def delete_taiga_hook(hook_id, token): TAIGA_API_URL = "https://api.taiga.io" url = f"{TAIGA_API_URL}/api/v1/webhooks/{hook_id}" - headers = { - "Authorization": f"Bearer {token}" - } + headers = {"Authorization": f"Bearer {token}"} resp = requests.delete(url, headers=headers) resp.raise_for_status() return resp - - - def delete_all_taiga_webhooks(token, webhook_url_taiga): - - mongo_client = pymongo.MongoClient(MONGO_URI) # URI to connect to the database - db = mongo_client[MONGO_DB] # Name of the database + mongo_client = pymongo.MongoClient(MONGO_URI) # URI to connect to the database + db = mongo_client[MONGO_DB] # Name of the database # We get all collections with 'epic' in their name and store them in a list. From them we will extract the projectsid of all the taiga projects active. all_collections = db.list_collection_names() taiga_collections = [c for c in all_collections if "epic" in c] - - #We iterate over the collections and the repositories to delete the webhooks - #We first get the repositories in the collection and then we get the hooks for each repository + + # We iterate over the collections and the repositories to delete the webhooks + # We first get the repositories in the collection and then we get the hooks for each repository for col_name in taiga_collections: distinct_project_ids = db[col_name].distinct("project_id") - + for project_id in distinct_project_ids: hooks = list_taiga_hooks(project_id, token) @@ -58,6 +50,10 @@ def delete_all_taiga_webhooks(token, webhook_url_taiga): if hook["url"] == webhook_url_taiga: try: delete_taiga_hook(hook["id"], token) - logger.info("Deleted taiga hook %s from project %s", hook['id'], project_id) + logger.info( + "Deleted taiga hook %s from project %s", + hook["id"], + project_id, + ) except requests.HTTPError as e: - logger.error("Error deleting taiga hook %s: %s", hook['id'], e) \ No newline at end of file + logger.error("Error deleting taiga hook %s: %s", hook["id"], e) From 53a79dea91331f802550bb08ec29f4fce6ecf6de Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Mon, 2 Mar 2026 16:07:04 +0100 Subject: [PATCH 04/32] feat: update Gitleaks installation method and run command in CI pipeline --- .github/workflows/ci-security-pipeline.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-security-pipeline.yml b/.github/workflows/ci-security-pipeline.yml index 27de289..01c78e3 100644 --- a/.github/workflows/ci-security-pipeline.yml +++ b/.github/workflows/ci-security-pipeline.yml @@ -88,8 +88,12 @@ jobs: with: fetch-depth: 0 + - name: Install Gitleaks (OSS) + run: | + curl -sSfL https://raw.githubusercontent.com/gitleaks/gitleaks/master/install.sh | sh -s -- -b /usr/local/bin + - name: Run Gitleaks - uses: gitleaks/gitleaks-action@v2 + run: gitleaks detect --source . --redact --verbose --exit-code 1 semgrep: runs-on: ubuntu-latest From 738033b6edaa76a65f73670aeb724d23ee421b10 Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Mon, 2 Mar 2026 16:14:18 +0100 Subject: [PATCH 05/32] fix: fixed bandit smells --- app.py | 7 ++++++- tests/test_settings.py | 5 +++-- utils/taiga_token/get_taiga_token.py | 3 ++- utils/webhook_deletion/delete_webhooks_github.py | 5 +++-- utils/webhook_deletion/delete_webhooks_taiga.py | 5 +++-- 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/app.py b/app.py index f0e5328..dbbd008 100644 --- a/app.py +++ b/app.py @@ -4,6 +4,7 @@ from routes.excel_routes import excel_bp from config.logger_config import setup_logging import logging +import os setup_logging() logger = logging.getLogger(__name__) @@ -22,4 +23,8 @@ def create_app(): if __name__ == "__main__": app = create_app() - app.run(debug=True, host="127.0.0.1", port=5000) + app.run( + debug=os.getenv("FLASK_DEBUG", "false").lower() == "true", + host="127.0.0.1", + port=5000, + ) diff --git a/tests/test_settings.py b/tests/test_settings.py index 383763d..85bf8cc 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1,6 +1,7 @@ """Tests for config/settings.py""" import os, importlib, pytest +from pathlib import Path from unittest.mock import patch import config.settings as settings_mod @@ -14,7 +15,7 @@ def test_require_env_missing_raises(self): "TAIGA_SIGNATURE_KEY": "x", "TAIGA_USERNAME": "x", "TAIGA_PASSWORD": "x", - "HOME": os.environ.get("HOME", "/tmp"), + "HOME": os.environ.get("HOME") or str(Path.home()), } with patch.dict(os.environ, env, clear=True), patch("dotenv.load_dotenv"): with pytest.raises(RuntimeError, match="TAIGA_API_URL"): @@ -88,7 +89,7 @@ def test_taiga_auth_url_defaults_to_api_url(self): "TAIGA_SIGNATURE_KEY": "ts", "TAIGA_USERNAME": "u", "TAIGA_PASSWORD": "p", - "HOME": os.environ.get("HOME", "/tmp"), + "HOME": os.environ.get("HOME") or str(Path.home()), } with patch.dict(os.environ, env, clear=True), patch("dotenv.load_dotenv"): importlib.reload(settings_mod) diff --git a/utils/taiga_token/get_taiga_token.py b/utils/taiga_token/get_taiga_token.py index a702d32..4e82f88 100644 --- a/utils/taiga_token/get_taiga_token.py +++ b/utils/taiga_token/get_taiga_token.py @@ -2,6 +2,7 @@ import requests logger = logging.getLogger(__name__) +REQUEST_TIMEOUT = (2, 10) def get_token(payload: dict) -> str: @@ -14,7 +15,7 @@ def get_token(payload: dict) -> str: login_url = "https://api.taiga.io/api/v1/auth" # Send the POST request to log in - response = requests.post(login_url, json=payload) + response = requests.post(login_url, json=payload, timeout=REQUEST_TIMEOUT) response.raise_for_status() # Will raise an error if the response status is not 200 # Parse the JSON response diff --git a/utils/webhook_deletion/delete_webhooks_github.py b/utils/webhook_deletion/delete_webhooks_github.py index a87deec..52f9770 100644 --- a/utils/webhook_deletion/delete_webhooks_github.py +++ b/utils/webhook_deletion/delete_webhooks_github.py @@ -10,6 +10,7 @@ ) logger = logging.getLogger(__name__) +REQUEST_TIMEOUT = 10 def list_github_hooks(owner, repo) -> list: @@ -23,7 +24,7 @@ def list_github_hooks(owner, repo) -> list: "Accept": "application/vnd.github.v3+json", "Authorization": f"Bearer {GITHUB_TOKEN}", } - resp = requests.get(url, headers=headers) + resp = requests.get(url, headers=headers, timeout=REQUEST_TIMEOUT) resp.raise_for_status() return resp.json() @@ -38,7 +39,7 @@ def delete_github_hook(owner, repo, hook_id): "Accept": "application/vnd.github.v3+json", "Authorization": f"Bearer {GITHUB_TOKEN}", } - resp = requests.delete(url, headers=headers) + resp = requests.delete(url, headers=headers, timeout=REQUEST_TIMEOUT) resp.raise_for_status() return resp diff --git a/utils/webhook_deletion/delete_webhooks_taiga.py b/utils/webhook_deletion/delete_webhooks_taiga.py index 44323af..7e9cd41 100644 --- a/utils/webhook_deletion/delete_webhooks_taiga.py +++ b/utils/webhook_deletion/delete_webhooks_taiga.py @@ -4,6 +4,7 @@ from config.settings import MONGO_URI, MONGO_DB, WEBHOOK_URL_TAIGA logger = logging.getLogger(__name__) +REQUEST_TIMEOUT = 10 # We define two functions to list and delete the webhooks using the API of github. @@ -11,7 +12,7 @@ def list_taiga_hooks(project_id, token): url = f"https://api.taiga.io/api/v1/webhooks?project={project_id}" headers = {"Authorization": f"Bearer {token}"} - resp = requests.get(url, headers=headers) + resp = requests.get(url, headers=headers, timeout=REQUEST_TIMEOUT) resp.raise_for_status() return resp.json() @@ -22,7 +23,7 @@ def delete_taiga_hook(hook_id, token): TAIGA_API_URL = "https://api.taiga.io" url = f"{TAIGA_API_URL}/api/v1/webhooks/{hook_id}" headers = {"Authorization": f"Bearer {token}"} - resp = requests.delete(url, headers=headers) + resp = requests.delete(url, headers=headers, timeout=REQUEST_TIMEOUT) resp.raise_for_status() return resp From f4e90fd24220ef3e262f1a74aef9a58b458cc3de Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Mon, 2 Mar 2026 16:40:27 +0100 Subject: [PATCH 06/32] feat: update Gitleaks installation method in CI pipeline --- .github/workflows/ci-security-pipeline.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-security-pipeline.yml b/.github/workflows/ci-security-pipeline.yml index 01c78e3..c44575f 100644 --- a/.github/workflows/ci-security-pipeline.yml +++ b/.github/workflows/ci-security-pipeline.yml @@ -88,9 +88,10 @@ jobs: with: fetch-depth: 0 - - name: Install Gitleaks (OSS) + - name: Install Gitleaks (via cli) run: | - curl -sSfL https://raw.githubusercontent.com/gitleaks/gitleaks/master/install.sh | sh -s -- -b /usr/local/bin + curl -sSL https://github.com/gitleaks/gitleaks/releases/latest/download/gitleaks-linux-amd64.tar.gz | tar xz -C /tmp + sudo mv /tmp/gitleaks-linux-amd64 /usr/local/bin/gitleaks - name: Run Gitleaks run: gitleaks detect --source . --redact --verbose --exit-code 1 From 789d0ec9b6d669a686ad1bb237b75347f4fdcbdc Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Mon, 2 Mar 2026 16:42:46 +0100 Subject: [PATCH 07/32] hotfix: now is the correct link --- .github/workflows/ci-security-pipeline.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-security-pipeline.yml b/.github/workflows/ci-security-pipeline.yml index c44575f..fd385cd 100644 --- a/.github/workflows/ci-security-pipeline.yml +++ b/.github/workflows/ci-security-pipeline.yml @@ -90,8 +90,10 @@ jobs: - name: Install Gitleaks (via cli) run: | - curl -sSL https://github.com/gitleaks/gitleaks/releases/latest/download/gitleaks-linux-amd64.tar.gz | tar xz -C /tmp - sudo mv /tmp/gitleaks-linux-amd64 /usr/local/bin/gitleaks + curl -sSL -o /tmp/gitleaks.tar.gz https://github.com/gitleaks/gitleaks/releases/download/v8.30.0/gitleaks_8.30.0_linux_x64.tar.gz + tar xz -C /tmp -f /tmp/gitleaks.tar.gz + sudo mv /tmp/gitleaks /usr/local/bin/gitleaks + sudo chmod +x /usr/local/bin/gitleaks - name: Run Gitleaks run: gitleaks detect --source . --redact --verbose --exit-code 1 From 839df9f80230e606221e2ca2186f56239d3cd05c Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Tue, 3 Mar 2026 11:22:41 +0100 Subject: [PATCH 08/32] ci: added baseline for gitleaks --- .github/workflows/ci-security-pipeline.yml | 2 +- .gitleaks-baseline.json | 443 +++++++++++++++++++++ 2 files changed, 444 insertions(+), 1 deletion(-) create mode 100644 .gitleaks-baseline.json diff --git a/.github/workflows/ci-security-pipeline.yml b/.github/workflows/ci-security-pipeline.yml index fd385cd..df31faa 100644 --- a/.github/workflows/ci-security-pipeline.yml +++ b/.github/workflows/ci-security-pipeline.yml @@ -96,7 +96,7 @@ jobs: sudo chmod +x /usr/local/bin/gitleaks - name: Run Gitleaks - run: gitleaks detect --source . --redact --verbose --exit-code 1 + run: gitleaks detect --source . --redact --verbose --baseline-path gitleaks-baseline.json --exit-code 1 semgrep: runs-on: ubuntu-latest diff --git a/.gitleaks-baseline.json b/.gitleaks-baseline.json new file mode 100644 index 0000000..afbd011 --- /dev/null +++ b/.gitleaks-baseline.json @@ -0,0 +1,443 @@ +[ + { + "RuleID": "github-pat", + "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "StartLine": 10, + "EndLine": 10, + "StartColumn": 23, + "EndColumn": 62, + "Match": "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", + "Secret": "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", + "File": "config_files/credentials_config.json", + "SymlinkFile": "", + "Commit": "51732615a354fe996d6892942eb0fd631c7570b3", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/51732615a354fe996d6892942eb0fd631c7570b3/config_files/credentials_config.json#L10", + "Entropy": 4.753056, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-07-01T14:36:18Z", + "Message": "Final modifications", + "Tags": [], + "Fingerprint": "51732615a354fe996d6892942eb0fd631c7570b3:config_files/credentials_config.json:github-pat:10" + }, + { + "RuleID": "jwt", + "Description": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", + "StartLine": 36, + "EndLine": 36, + "StartColumn": 11, + "EndColumn": 522, + "Match": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQ3MzMxMDk0LCJqdGkiOiI4N2U1NWM5MTNmMzU0NjQwODU1MTFiMWMwY2I0YzM3NSIsInVzZXJfaWQiOjc1OTg5Nn0.Rh57RIdpOZLQ_xg_5c22dLIp3yXvvWB-aC2RVwgLdvSOirkhIBgqDKBXQ3j3OlbEi4BIgD-WfZZR6CXtyewOnX8ov4RCtTtdpxpuo8lch4ZhqPuvZ-UT-w8ytenrcxZoH3vz3ikUaevYDbjuCV3FSoiWn1Xxcg8jdiu-bsx-nenZ7GhydvE6VCKogF29bPjLUuZbkk-BtxVHiTPDEb6qOWx7wo83b4Io8D0zaKxgVQRzliUUy-my8HdWTex-ELyaIwzWVAkzbYGh7DjmRY4opGqFovmDkOCCOmv8Ycm3VU2RqFd7nfJAEZayMkwe1l481dvmPKMfcJ0llORocbkC0A\"", + "Secret": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQ3MzMxMDk0LCJqdGkiOiI4N2U1NWM5MTNmMzU0NjQwODU1MTFiMWMwY2I0YzM3NSIsInVzZXJfaWQiOjc1OTg5Nn0.Rh57RIdpOZLQ_xg_5c22dLIp3yXvvWB-aC2RVwgLdvSOirkhIBgqDKBXQ3j3OlbEi4BIgD-WfZZR6CXtyewOnX8ov4RCtTtdpxpuo8lch4ZhqPuvZ-UT-w8ytenrcxZoH3vz3ikUaevYDbjuCV3FSoiWn1Xxcg8jdiu-bsx-nenZ7GhydvE6VCKogF29bPjLUuZbkk-BtxVHiTPDEb6qOWx7wo83b4Io8D0zaKxgVQRzliUUy-my8HdWTex-ELyaIwzWVAkzbYGh7DjmRY4opGqFovmDkOCCOmv8Ycm3VU2RqFd7nfJAEZayMkwe1l481dvmPKMfcJ0llORocbkC0A", + "File": "utils/taiga_get_milestone_points.py", + "SymlinkFile": "", + "Commit": "81c19ba5b4da57e7b234a97a75369c537a122880", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/81c19ba5b4da57e7b234a97a75369c537a122880/utils/taiga_get_milestone_points.py#L36", + "Entropy": 5.883454, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-05-19T23:42:51Z", + "Message": "milestone_points", + "Tags": [], + "Fingerprint": "81c19ba5b4da57e7b234a97a75369c537a122880:utils/taiga_get_milestone_points.py:jwt:36" + }, + { + "RuleID": "jwt", + "Description": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", + "StartLine": 37, + "EndLine": 37, + "StartColumn": 11, + "EndColumn": 522, + "Match": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQ3MzA3MDI1LCJqdGkiOiIwMzA5NDcyNzZlMjc0YzQxYWM0N2NiYTllNzE2NGYyMyIsInVzZXJfaWQiOjc1OTg5Nn0.bYh7cfGv6WzhpdQB1nXu75oEI3uklWvEQAkoItY5R9j0WPutzVcXbAbeXkBH-sfJa6k-QjVCKfFYruqGSH4819q7tCYzc67sqgiA0MTsJkKuzUa_2aa1owyHMtiDYGMK3ZhN8W7KQxarMEvHXEtyrUKBI5gU_ewUoqtLBcplCBSFfrIvC9UqjA7OzS3YlBaS9YCYP3vt0ndg_qGRq1hqb64sByx7eld_6z-1Rm2KmfqHztqj6miR4K3KhNlO45lyNti_WE4nZJxfku4yXo8G91MSVFQSCZgLXUvRL-DO0b28Fu6LQX5T9or3cDNvuaGuT0SP3A1Jf-KIYU21sWYnrA\"", + "Secret": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQ3MzA3MDI1LCJqdGkiOiIwMzA5NDcyNzZlMjc0YzQxYWM0N2NiYTllNzE2NGYyMyIsInVzZXJfaWQiOjc1OTg5Nn0.bYh7cfGv6WzhpdQB1nXu75oEI3uklWvEQAkoItY5R9j0WPutzVcXbAbeXkBH-sfJa6k-QjVCKfFYruqGSH4819q7tCYzc67sqgiA0MTsJkKuzUa_2aa1owyHMtiDYGMK3ZhN8W7KQxarMEvHXEtyrUKBI5gU_ewUoqtLBcplCBSFfrIvC9UqjA7OzS3YlBaS9YCYP3vt0ndg_qGRq1hqb64sByx7eld_6z-1Rm2KmfqHztqj6miR4K3KhNlO45lyNti_WE4nZJxfku4yXo8G91MSVFQSCZgLXUvRL-DO0b28Fu6LQX5T9or3cDNvuaGuT0SP3A1Jf-KIYU21sWYnrA", + "File": "test_points.py", + "SymlinkFile": "", + "Commit": "81c19ba5b4da57e7b234a97a75369c537a122880", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/81c19ba5b4da57e7b234a97a75369c537a122880/test_points.py#L37", + "Entropy": 5.9098024, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-05-19T23:42:51Z", + "Message": "milestone_points", + "Tags": [], + "Fingerprint": "81c19ba5b4da57e7b234a97a75369c537a122880:test_points.py:jwt:37" + }, + { + "RuleID": "github-pat", + "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "StartLine": 5, + "EndLine": 5, + "StartColumn": 22, + "EndColumn": 61, + "Match": "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", + "Secret": "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", + "File": "test.py", + "SymlinkFile": "", + "Commit": "4c29834f8242fe60c28182f982df7c9616923cf4", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/4c29834f8242fe60c28182f982df7c9616923cf4/test.py#L5", + "Entropy": 4.803056, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-05-27T00:26:14Z", + "Message": "recovry", + "Tags": [], + "Fingerprint": "4c29834f8242fe60c28182f982df7c9616923cf4:test.py:github-pat:5" + }, + { + "RuleID": "github-pat", + "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "StartLine": 40, + "EndLine": 40, + "StartColumn": 11, + "EndColumn": 50, + "Match": "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", + "Secret": "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", + "File": "test.py", + "SymlinkFile": "", + "Commit": "4c29834f8242fe60c28182f982df7c9616923cf4", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/4c29834f8242fe60c28182f982df7c9616923cf4/test.py#L40", + "Entropy": 4.803056, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-05-27T00:26:14Z", + "Message": "recovry", + "Tags": [], + "Fingerprint": "4c29834f8242fe60c28182f982df7c9616923cf4:test.py:github-pat:40" + }, + { + "RuleID": "github-pat", + "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "StartLine": 12, + "EndLine": 12, + "StartColumn": 18, + "EndColumn": 57, + "Match": "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", + "Secret": "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", + "File": "env", + "SymlinkFile": "", + "Commit": "4bbb59be36182bdab8829924c311a6109bc30748", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/4bbb59be36182bdab8829924c311a6109bc30748/env#L12", + "Entropy": 4.8341837, + "Author": "Pablo", + "Email": "pgomezna@gmail.com", + "Date": "2025-04-04T17:32:22Z", + "Message": "API change", + "Tags": [], + "Fingerprint": "4bbb59be36182bdab8829924c311a6109bc30748:env:github-pat:12" + }, + { + "RuleID": "jwt", + "Description": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", + "StartLine": 17, + "EndLine": 17, + "StartColumn": 17, + "EndColumn": 528, + "Match": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQyOTQxMDg1LCJqdGkiOiJiMzJhZjE1NDU4NzI0OThhOTY2NWJhMWVkOTIxZjMzYSIsInVzZXJfaWQiOjc1OTg5Nn0.H8ug241uZP6q2AbSKSZZbcxRSlRE52wGwok8FfZGhVjnshGqGkbEJnhodESqI94bT6bPCToXxDeLqLEdwQDMz7t5j7vDioxyG8cxF2E2bjLVLt6-wkT6U1mdomyyYvpP78gkmlNPr0a3Jv60MplTgcSnfrQ11N2xYDzf1yNtTJsYCjZKzy5d9zqsOkILBKpqnpVu5FcJlXdYyc-K0TMNsWbi0zsA3F7OBchEfOREZcdwZNfN7Qv4s5pb38ZdPQsfD_wGrvjzgE5FdOOvV7-9gEREI7ruytTawsoHSLMd_lwxIfXBUUJxJSAb13oDKEGWY5N9fVN0B3ayuRtwjZF-nw\"", + "Secret": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQyOTQxMDg1LCJqdGkiOiJiMzJhZjE1NDU4NzI0OThhOTY2NWJhMWVkOTIxZjMzYSIsInVzZXJfaWQiOjc1OTg5Nn0.H8ug241uZP6q2AbSKSZZbcxRSlRE52wGwok8FfZGhVjnshGqGkbEJnhodESqI94bT6bPCToXxDeLqLEdwQDMz7t5j7vDioxyG8cxF2E2bjLVLt6-wkT6U1mdomyyYvpP78gkmlNPr0a3Jv60MplTgcSnfrQ11N2xYDzf1yNtTJsYCjZKzy5d9zqsOkILBKpqnpVu5FcJlXdYyc-K0TMNsWbi0zsA3F7OBchEfOREZcdwZNfN7Qv4s5pb38ZdPQsfD_wGrvjzgE5FdOOvV7-9gEREI7ruytTawsoHSLMd_lwxIfXBUUJxJSAb13oDKEGWY5N9fVN0B3ayuRtwjZF-nw", + "File": "env", + "SymlinkFile": "", + "Commit": "4bbb59be36182bdab8829924c311a6109bc30748", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/4bbb59be36182bdab8829924c311a6109bc30748/env#L17", + "Entropy": 5.886606, + "Author": "Pablo", + "Email": "pgomezna@gmail.com", + "Date": "2025-04-04T17:32:22Z", + "Message": "API change", + "Tags": [], + "Fingerprint": "4bbb59be36182bdab8829924c311a6109bc30748:env:jwt:17" + }, + { + "RuleID": "jwt", + "Description": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", + "StartLine": 9, + "EndLine": 9, + "StartColumn": 11, + "EndColumn": 522, + "Match": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQzNzA4Nzg1LCJqdGkiOiJlNDlhNjgwMjQ1YzA0ZjI1OTM2YWQxMjgyNWZjYTgyNiIsInVzZXJfaWQiOjc1OTg5Nn0.FkmK4xi260OzsyXqvqOyV6DyzTysqUkBp4Vefm_267VKXgHwuTY_dKgVKjcjhYyDkMQWGDvx0v2NOz8Eamx2RKo81jZtb0vgmHBWwQy9pse8Xj5PJIFrDy0znQvF0gMc5Jwr-ebAkiZizufg7E4a4P4T8sEm2OYauup89769TZDWHcHfE3iBoDWW24J2x5m6PlCJZQM-Ro4DZPhXJezAoCi8pewG56VUp02LsNo_yW1S-vB1n3V6xcaEJjKaYLx8iKuQ4FtgMZB6wcCGD-HD9aZmrFfrQf_wi5qCuIz0hf8gMJC33OwGrlsTDr626Xb_qpbnABRLTvxjY0D0NnLq8A'", + "Secret": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQzNzA4Nzg1LCJqdGkiOiJlNDlhNjgwMjQ1YzA0ZjI1OTM2YWQxMjgyNWZjYTgyNiIsInVzZXJfaWQiOjc1OTg5Nn0.FkmK4xi260OzsyXqvqOyV6DyzTysqUkBp4Vefm_267VKXgHwuTY_dKgVKjcjhYyDkMQWGDvx0v2NOz8Eamx2RKo81jZtb0vgmHBWwQy9pse8Xj5PJIFrDy0znQvF0gMc5Jwr-ebAkiZizufg7E4a4P4T8sEm2OYauup89769TZDWHcHfE3iBoDWW24J2x5m6PlCJZQM-Ro4DZPhXJezAoCi8pewG56VUp02LsNo_yW1S-vB1n3V6xcaEJjKaYLx8iKuQ4FtgMZB6wcCGD-HD9aZmrFfrQf_wi5qCuIz0hf8gMJC33OwGrlsTDr626Xb_qpbnABRLTvxjY0D0NnLq8A", + "File": "utils/members.py", + "SymlinkFile": "", + "Commit": "e13694037199e3bea3de23ae0131e636c62d5aed", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/e13694037199e3bea3de23ae0131e636c62d5aed/utils/members.py#L9", + "Entropy": 5.888001, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-04-04T10:01:44Z", + "Message": "token modifications", + "Tags": [], + "Fingerprint": "e13694037199e3bea3de23ae0131e636c62d5aed:utils/members.py:jwt:9" + }, + { + "RuleID": "github-pat", + "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "StartLine": 24, + "EndLine": 24, + "StartColumn": 20, + "EndColumn": 59, + "Match": "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", + "Secret": "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", + "File": "recovery/github_recovery.py", + "SymlinkFile": "", + "Commit": "4c29834f8242fe60c28182f982df7c9616923cf4", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/4c29834f8242fe60c28182f982df7c9616923cf4/recovery/github_recovery.py#L24", + "Entropy": 4.803056, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-05-27T00:26:14Z", + "Message": "recovry", + "Tags": [], + "Fingerprint": "4c29834f8242fe60c28182f982df7c9616923cf4:recovery/github_recovery.py:github-pat:24" + }, + { + "RuleID": "github-pat", + "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "StartLine": 10, + "EndLine": 10, + "StartColumn": 44, + "EndColumn": 83, + "Match": "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", + "Secret": "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", + "File": "datasources/github_handler.py", + "SymlinkFile": "", + "Commit": "fb5af68da7c610cb306db0db52fdae2473fa7a28", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/fb5af68da7c610cb306db0db52fdae2473fa7a28/datasources/github_handler.py#L10", + "Entropy": 4.8341837, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-26T15:54:51Z", + "Message": "Solving problems with API call to commit stats", + "Tags": [], + "Fingerprint": "fb5af68da7c610cb306db0db52fdae2473fa7a28:datasources/github_handler.py:github-pat:10" + }, + { + "RuleID": "github-pat", + "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "StartLine": 7, + "EndLine": 7, + "StartColumn": 28, + "EndColumn": 67, + "Match": "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", + "Secret": "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", + "File": "datasources/github_handler.py", + "SymlinkFile": "", + "Commit": "429d462f6209cc32022bb0d8ecab3052c16ed1d7", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/429d462f6209cc32022bb0d8ecab3052c16ed1d7/datasources/github_handler.py#L7", + "Entropy": 4.8341837, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-26T13:02:39Z", + "Message": "Commit Stats First implementation", + "Tags": [], + "Fingerprint": "429d462f6209cc32022bb0d8ecab3052c16ed1d7:datasources/github_handler.py:github-pat:7" + }, + { + "RuleID": "github-pat", + "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "StartLine": 7, + "EndLine": 7, + "StartColumn": 72, + "EndColumn": 111, + "Match": "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", + "Secret": "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", + "File": "datasources/github_handler.py", + "SymlinkFile": "", + "Commit": "429d462f6209cc32022bb0d8ecab3052c16ed1d7", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/429d462f6209cc32022bb0d8ecab3052c16ed1d7/datasources/github_handler.py#L7", + "Entropy": 4.8341837, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-26T13:02:39Z", + "Message": "Commit Stats First implementation", + "Tags": [], + "Fingerprint": "429d462f6209cc32022bb0d8ecab3052c16ed1d7:datasources/github_handler.py:github-pat:7" + }, + { + "RuleID": "github-pat", + "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "StartLine": 6, + "EndLine": 6, + "StartColumn": 28, + "EndColumn": 67, + "Match": "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", + "Secret": "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", + "File": "utils/delete_webhooks_github.py", + "SymlinkFile": "", + "Commit": "468587ecc60805fad0da7d4c8201705392cfd188", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/468587ecc60805fad0da7d4c8201705392cfd188/utils/delete_webhooks_github.py#L6", + "Entropy": 4.8341837, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-26T10:05:46Z", + "Message": "Script fot Deletion of Webhooks", + "Tags": [], + "Fingerprint": "468587ecc60805fad0da7d4c8201705392cfd188:utils/delete_webhooks_github.py:github-pat:6" + }, + { + "RuleID": "github-pat", + "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "StartLine": 6, + "EndLine": 6, + "StartColumn": 72, + "EndColumn": 111, + "Match": "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", + "Secret": "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", + "File": "utils/delete_webhooks_github.py", + "SymlinkFile": "", + "Commit": "468587ecc60805fad0da7d4c8201705392cfd188", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/468587ecc60805fad0da7d4c8201705392cfd188/utils/delete_webhooks_github.py#L6", + "Entropy": 4.8341837, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-26T10:05:46Z", + "Message": "Script fot Deletion of Webhooks", + "Tags": [], + "Fingerprint": "468587ecc60805fad0da7d4c8201705392cfd188:utils/delete_webhooks_github.py:github-pat:6" + }, + { + "RuleID": "jwt", + "Description": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", + "StartLine": 10, + "EndLine": 10, + "StartColumn": 27, + "EndColumn": 538, + "Match": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQyOTQxMDg1LCJqdGkiOiJiMzJhZjE1NDU4NzI0OThhOTY2NWJhMWVkOTIxZjMzYSIsInVzZXJfaWQiOjc1OTg5Nn0.H8ug241uZP6q2AbSKSZZbcxRSlRE52wGwok8FfZGhVjnshGqGkbEJnhodESqI94bT6bPCToXxDeLqLEdwQDMz7t5j7vDioxyG8cxF2E2bjLVLt6-wkT6U1mdomyyYvpP78gkmlNPr0a3Jv60MplTgcSnfrQ11N2xYDzf1yNtTJsYCjZKzy5d9zqsOkILBKpqnpVu5FcJlXdYyc-K0TMNsWbi0zsA3F7OBchEfOREZcdwZNfN7Qv4s5pb38ZdPQsfD_wGrvjzgE5FdOOvV7-9gEREI7ruytTawsoHSLMd_lwxIfXBUUJxJSAb13oDKEGWY5N9fVN0B3ayuRtwjZF-nw\"", + "Secret": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQyOTQxMDg1LCJqdGkiOiJiMzJhZjE1NDU4NzI0OThhOTY2NWJhMWVkOTIxZjMzYSIsInVzZXJfaWQiOjc1OTg5Nn0.H8ug241uZP6q2AbSKSZZbcxRSlRE52wGwok8FfZGhVjnshGqGkbEJnhodESqI94bT6bPCToXxDeLqLEdwQDMz7t5j7vDioxyG8cxF2E2bjLVLt6-wkT6U1mdomyyYvpP78gkmlNPr0a3Jv60MplTgcSnfrQ11N2xYDzf1yNtTJsYCjZKzy5d9zqsOkILBKpqnpVu5FcJlXdYyc-K0TMNsWbi0zsA3F7OBchEfOREZcdwZNfN7Qv4s5pb38ZdPQsfD_wGrvjzgE5FdOOvV7-9gEREI7ruytTawsoHSLMd_lwxIfXBUUJxJSAb13oDKEGWY5N9fVN0B3ayuRtwjZF-nw", + "File": "utils/delete_webhooks_taiga.py", + "SymlinkFile": "", + "Commit": "468587ecc60805fad0da7d4c8201705392cfd188", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/468587ecc60805fad0da7d4c8201705392cfd188/utils/delete_webhooks_taiga.py#L10", + "Entropy": 5.886606, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-26T10:05:46Z", + "Message": "Script fot Deletion of Webhooks", + "Tags": [], + "Fingerprint": "468587ecc60805fad0da7d4c8201705392cfd188:utils/delete_webhooks_taiga.py:jwt:10" + }, + { + "RuleID": "jwt", + "Description": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", + "StartLine": 10, + "EndLine": 10, + "StartColumn": 542, + "EndColumn": 1053, + "Match": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQyOTQxMDg1LCJqdGkiOiJiMzJhZjE1NDU4NzI0OThhOTY2NWJhMWVkOTIxZjMzYSIsInVzZXJfaWQiOjc1OTg5Nn0.H8ug241uZP6q2AbSKSZZbcxRSlRE52wGwok8FfZGhVjnshGqGkbEJnhodESqI94bT6bPCToXxDeLqLEdwQDMz7t5j7vDioxyG8cxF2E2bjLVLt6-wkT6U1mdomyyYvpP78gkmlNPr0a3Jv60MplTgcSnfrQ11N2xYDzf1yNtTJsYCjZKzy5d9zqsOkILBKpqnpVu5FcJlXdYyc-K0TMNsWbi0zsA3F7OBchEfOREZcdwZNfN7Qv4s5pb38ZdPQsfD_wGrvjzgE5FdOOvV7-9gEREI7ruytTawsoHSLMd_lwxIfXBUUJxJSAb13oDKEGWY5N9fVN0B3ayuRtwjZF-nw\"", + "Secret": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQyOTQxMDg1LCJqdGkiOiJiMzJhZjE1NDU4NzI0OThhOTY2NWJhMWVkOTIxZjMzYSIsInVzZXJfaWQiOjc1OTg5Nn0.H8ug241uZP6q2AbSKSZZbcxRSlRE52wGwok8FfZGhVjnshGqGkbEJnhodESqI94bT6bPCToXxDeLqLEdwQDMz7t5j7vDioxyG8cxF2E2bjLVLt6-wkT6U1mdomyyYvpP78gkmlNPr0a3Jv60MplTgcSnfrQ11N2xYDzf1yNtTJsYCjZKzy5d9zqsOkILBKpqnpVu5FcJlXdYyc-K0TMNsWbi0zsA3F7OBchEfOREZcdwZNfN7Qv4s5pb38ZdPQsfD_wGrvjzgE5FdOOvV7-9gEREI7ruytTawsoHSLMd_lwxIfXBUUJxJSAb13oDKEGWY5N9fVN0B3ayuRtwjZF-nw", + "File": "utils/delete_webhooks_taiga.py", + "SymlinkFile": "", + "Commit": "468587ecc60805fad0da7d4c8201705392cfd188", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/468587ecc60805fad0da7d4c8201705392cfd188/utils/delete_webhooks_taiga.py#L10", + "Entropy": 5.886606, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-26T10:05:46Z", + "Message": "Script fot Deletion of Webhooks", + "Tags": [], + "Fingerprint": "468587ecc60805fad0da7d4c8201705392cfd188:utils/delete_webhooks_taiga.py:jwt:10" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 56, + "EndLine": 56, + "StartColumn": 6, + "EndColumn": 35, + "Match": "key: ed25519.Ed25519PrivateKey", + "Secret": "ed25519.Ed25519PrivateKey", + "File": "myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py", + "SymlinkFile": "", + "Commit": "13add36a5bbcb3e039f0fea45dc1188ab9747a36", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/13add36a5bbcb3e039f0fea45dc1188ab9747a36/myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py#L56", + "Entropy": 3.8136606, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-24T19:14:14Z", + "Message": "LD_Connect Working", + "Tags": [], + "Fingerprint": "13add36a5bbcb3e039f0fea45dc1188ab9747a36:myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py:generic-api-key:56" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 57, + "EndLine": 57, + "StartColumn": 6, + "EndColumn": 40, + "Match": "key_cls = ed25519.Ed25519PrivateKey", + "Secret": "ed25519.Ed25519PrivateKey", + "File": "myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py", + "SymlinkFile": "", + "Commit": "13add36a5bbcb3e039f0fea45dc1188ab9747a36", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/13add36a5bbcb3e039f0fea45dc1188ab9747a36/myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py#L57", + "Entropy": 3.8136606, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-24T19:14:14Z", + "Message": "LD_Connect Working", + "Tags": [], + "Fingerprint": "13add36a5bbcb3e039f0fea45dc1188ab9747a36:myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py:generic-api-key:57" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 68, + "EndLine": 68, + "StartColumn": 6, + "EndColumn": 31, + "Match": "key: ed448.Ed448PrivateKey", + "Secret": "ed448.Ed448PrivateKey", + "File": "myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py", + "SymlinkFile": "", + "Commit": "13add36a5bbcb3e039f0fea45dc1188ab9747a36", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/13add36a5bbcb3e039f0fea45dc1188ab9747a36/myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py#L68", + "Entropy": 3.5944657, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-24T19:14:14Z", + "Message": "LD_Connect Working", + "Tags": [], + "Fingerprint": "13add36a5bbcb3e039f0fea45dc1188ab9747a36:myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py:generic-api-key:68" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 69, + "EndLine": 69, + "StartColumn": 6, + "EndColumn": 36, + "Match": "key_cls = ed448.Ed448PrivateKey", + "Secret": "ed448.Ed448PrivateKey", + "File": "myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py", + "SymlinkFile": "", + "Commit": "13add36a5bbcb3e039f0fea45dc1188ab9747a36", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/13add36a5bbcb3e039f0fea45dc1188ab9747a36/myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py#L69", + "Entropy": 3.5944657, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-24T19:14:14Z", + "Message": "LD_Connect Working", + "Tags": [], + "Fingerprint": "13add36a5bbcb3e039f0fea45dc1188ab9747a36:myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py:generic-api-key:69" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 43, + "EndLine": 43, + "StartColumn": 10, + "EndColumn": 41, + "Match": "RegEnumKey = win32api.RegEnumKey", + "Secret": "win32api.RegEnumKey", + "File": "myenv/Lib/site-packages/setuptools/_distutils/msvccompiler.py", + "SymlinkFile": "", + "Commit": "13add36a5bbcb3e039f0fea45dc1188ab9747a36", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/13add36a5bbcb3e039f0fea45dc1188ab9747a36/myenv/Lib/site-packages/setuptools/_distutils/msvccompiler.py#L43", + "Entropy": 3.932138, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-24T19:14:14Z", + "Message": "LD_Connect Working", + "Tags": [], + "Fingerprint": "13add36a5bbcb3e039f0fea45dc1188ab9747a36:myenv/Lib/site-packages/setuptools/_distutils/msvccompiler.py:generic-api-key:43" + } +] From bf8493c6983d0f6ab77df90e4a025d425db2fc1d Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Tue, 3 Mar 2026 11:23:55 +0100 Subject: [PATCH 09/32] docs: fixed how to run tests --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index d7531df..c11df9c 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,6 @@ All endpoints return **`200 OK`** immediately; heavy work continues asynchronou ```bash pytest # unit tests -locust -f tests/ # stress tests (replay real‑world payloads) ``` --- From fc7a6879b0d51d38e9337205d3744ec076fd2a14 Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Tue, 3 Mar 2026 11:26:09 +0100 Subject: [PATCH 10/32] ci: added missing dot --- .github/workflows/ci-security-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-security-pipeline.yml b/.github/workflows/ci-security-pipeline.yml index df31faa..d9b1c8a 100644 --- a/.github/workflows/ci-security-pipeline.yml +++ b/.github/workflows/ci-security-pipeline.yml @@ -96,7 +96,7 @@ jobs: sudo chmod +x /usr/local/bin/gitleaks - name: Run Gitleaks - run: gitleaks detect --source . --redact --verbose --baseline-path gitleaks-baseline.json --exit-code 1 + run: gitleaks detect --source . --redact --verbose --baseline-path .gitleaks-baseline.json --exit-code 1 semgrep: runs-on: ubuntu-latest From 6a1650ae574e7abc991712e8079503b4d5149a1b Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Tue, 3 Mar 2026 12:13:10 +0100 Subject: [PATCH 11/32] style: fixed ruff --- .gitignore | 1 + config/credentials_loader.py | 3 ++- datasources/excel_handler.py | 1 - datasources/github_handler.py | 1 - routes/excel_routes.py | 1 - tests/conftest.py | 4 +++- tests/test_api_event_publisher.py | 1 - tests/test_app.py | 1 - tests/test_credentials_loader.py | 3 ++- tests/test_datetime_utils.py | 1 - tests/test_delete_webhooks_github.py | 1 - tests/test_delete_webhooks_taiga.py | 1 - tests/test_excel_handler.py | 1 - tests/test_excel_routes.py | 2 +- tests/test_github_api_call.py | 2 -- tests/test_github_handler.py | 3 +-- tests/test_github_recovery.py | 2 -- tests/test_github_routes.py | 4 +++- tests/test_logger_config.py | 3 ++- tests/test_mongo_client.py | 1 - tests/test_settings.py | 4 +++- tests/test_taiga_api_call.py | 2 -- tests/test_taiga_auth.py | 3 ++- tests/test_taiga_handler.py | 14 ++++++++++---- tests/test_taiga_routes.py | 4 +++- tests/test_verify_signature_github.py | 3 ++- tests/test_verify_signature_taiga.py | 3 ++- utils/recovery/github_recovery.py | 3 ++- utils/recovery/taiga_recovery.py | 7 ++++--- utils/taiga_token/taiga_auth.py | 4 +++- utils/webhook_deletion/delete_webhooks_github.py | 1 - utils/webhook_deletion/delete_webhooks_taiga.py | 2 +- 32 files changed, 47 insertions(+), 40 deletions(-) diff --git a/.gitignore b/.gitignore index 695768a..75dd118 100644 --- a/.gitignore +++ b/.gitignore @@ -217,3 +217,4 @@ __marimo__/ # custom config_files/credentials_config.json +.vscode/mcp.json diff --git a/config/credentials_loader.py b/config/credentials_loader.py index 6a992c9..0f6c971 100644 --- a/config/credentials_loader.py +++ b/config/credentials_loader.py @@ -1,4 +1,5 @@ -import json, os +import json +import os from typing import Optional CONFIG_FILE = os.getenv("CREDENTIALS_FILE", "config_files/credentials_config.json") diff --git a/datasources/excel_handler.py b/datasources/excel_handler.py index cb88b24..f422c2d 100644 --- a/datasources/excel_handler.py +++ b/datasources/excel_handler.py @@ -1,4 +1,3 @@ -from datetime import datetime # List of activity types extracted from the Excel sheet diff --git a/datasources/github_handler.py b/datasources/github_handler.py index 470568e..e92b281 100644 --- a/datasources/github_handler.py +++ b/datasources/github_handler.py @@ -1,7 +1,6 @@ from typing import Dict import re from datasources.requests.github_api_call import fetch_commit_stats -from config.credentials_loader import resolve from utils.datetime_utils import to_madrid_local diff --git a/routes/excel_routes.py b/routes/excel_routes.py index f9af451..8033dbc 100644 --- a/routes/excel_routes.py +++ b/routes/excel_routes.py @@ -1,7 +1,6 @@ from flask import Blueprint, request, jsonify from datasources.excel_handler import parse_excel_event from database.mongo_client import get_collection -from routes.API_publisher.API_event_publisher import notify_eval_push from config.logger_config import setup_logging import logging diff --git a/tests/conftest.py b/tests/conftest.py index c378ad9..6e7d109 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,9 @@ Shared pytest fixtures for the LD_Connect_Event test suite. """ -import os, json, pytest +import os +import json +import pytest # ── Set required env vars BEFORE any application module is imported ────────── os.environ.setdefault("GITHUB_SIGNATURE_KEY", "test-github-secret") diff --git a/tests/test_api_event_publisher.py b/tests/test_api_event_publisher.py index 5df4076..f06f4a7 100644 --- a/tests/test_api_event_publisher.py +++ b/tests/test_api_event_publisher.py @@ -1,6 +1,5 @@ """Tests for routes/API_publisher/API_event_publisher.py""" -import pytest from unittest.mock import patch, MagicMock diff --git a/tests/test_app.py b/tests/test_app.py index d19953d..cf82a91 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -1,6 +1,5 @@ """Tests for app.py""" -import pytest class TestCreateApp: diff --git a/tests/test_credentials_loader.py b/tests/test_credentials_loader.py index f2daf74..a24ccd4 100644 --- a/tests/test_credentials_loader.py +++ b/tests/test_credentials_loader.py @@ -1,6 +1,7 @@ """Tests for config/credentials_loader.py""" -import json, os, pytest +import json +import pytest from unittest.mock import patch diff --git a/tests/test_datetime_utils.py b/tests/test_datetime_utils.py index 05e0554..75d4f8d 100644 --- a/tests/test_datetime_utils.py +++ b/tests/test_datetime_utils.py @@ -1,6 +1,5 @@ """Tests for utils/datetime_utils.py""" -import pytest from utils.datetime_utils import to_madrid_local diff --git a/tests/test_delete_webhooks_github.py b/tests/test_delete_webhooks_github.py index fa68bb5..7a51eae 100644 --- a/tests/test_delete_webhooks_github.py +++ b/tests/test_delete_webhooks_github.py @@ -1,6 +1,5 @@ """Tests for utils/webhook_deletion/delete_webhooks_github.py""" -import pytest from unittest.mock import patch, MagicMock diff --git a/tests/test_delete_webhooks_taiga.py b/tests/test_delete_webhooks_taiga.py index 434fc6c..782a04f 100644 --- a/tests/test_delete_webhooks_taiga.py +++ b/tests/test_delete_webhooks_taiga.py @@ -1,6 +1,5 @@ """Tests for utils/webhook_deletion/delete_webhooks_taiga.py""" -import pytest from unittest.mock import patch, MagicMock diff --git a/tests/test_excel_handler.py b/tests/test_excel_handler.py index ece9a73..00bb04b 100644 --- a/tests/test_excel_handler.py +++ b/tests/test_excel_handler.py @@ -1,6 +1,5 @@ """Tests for datasources/excel_handler.py""" -import pytest from datasources.excel_handler import parse_excel_event, ACTIVITY_TYPES diff --git a/tests/test_excel_routes.py b/tests/test_excel_routes.py index 6b0fe9f..3e856ee 100644 --- a/tests/test_excel_routes.py +++ b/tests/test_excel_routes.py @@ -1,6 +1,6 @@ """Tests for routes/excel_routes.py""" -import json, pytest +import json from unittest.mock import patch, MagicMock diff --git a/tests/test_github_api_call.py b/tests/test_github_api_call.py index 435e4b7..6ce58c1 100644 --- a/tests/test_github_api_call.py +++ b/tests/test_github_api_call.py @@ -1,6 +1,5 @@ """Tests for datasources/requests/github_api_call.py""" -import pytest from unittest.mock import patch, MagicMock @@ -50,7 +49,6 @@ def test_network_error_returns_zeros(self, mock_get, mock_resolve): def test_uses_correct_url(self, mock_get, mock_resolve): from datasources.requests.github_api_call import ( fetch_commit_stats, - GITHUB_API_URL, ) mock_resp = MagicMock() diff --git a/tests/test_github_handler.py b/tests/test_github_handler.py index 03d85a0..b1c7ff8 100644 --- a/tests/test_github_handler.py +++ b/tests/test_github_handler.py @@ -1,7 +1,6 @@ """Tests for datasources/github_handler.py""" -import pytest -from unittest.mock import patch, MagicMock +from unittest.mock import patch class TestParseGithubEvent: diff --git a/tests/test_github_recovery.py b/tests/test_github_recovery.py index 2388cc8..f018d5e 100644 --- a/tests/test_github_recovery.py +++ b/tests/test_github_recovery.py @@ -1,8 +1,6 @@ """Tests for utils/recovery/github_recovery.py""" -import pytest from unittest.mock import patch, MagicMock -from datetime import datetime class TestParseDt: diff --git a/tests/test_github_routes.py b/tests/test_github_routes.py index 81b3cd7..7955961 100644 --- a/tests/test_github_routes.py +++ b/tests/test_github_routes.py @@ -1,6 +1,8 @@ """Tests for routes/github_routes.py""" -import hashlib, hmac, json, pytest +import hashlib +import hmac +import json from unittest.mock import patch, MagicMock diff --git a/tests/test_logger_config.py b/tests/test_logger_config.py index ffc96e4..5eeca78 100644 --- a/tests/test_logger_config.py +++ b/tests/test_logger_config.py @@ -1,6 +1,7 @@ """Tests for config/logger_config.py""" -import logging, os, pytest +import logging +import os from unittest.mock import patch diff --git a/tests/test_mongo_client.py b/tests/test_mongo_client.py index deffc7d..00a7156 100644 --- a/tests/test_mongo_client.py +++ b/tests/test_mongo_client.py @@ -1,6 +1,5 @@ """Tests for database/mongo_client.py""" -import pytest from unittest.mock import patch, MagicMock diff --git a/tests/test_settings.py b/tests/test_settings.py index 85bf8cc..dc84976 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1,6 +1,8 @@ """Tests for config/settings.py""" -import os, importlib, pytest +import os +import importlib +import pytest from pathlib import Path from unittest.mock import patch diff --git a/tests/test_taiga_api_call.py b/tests/test_taiga_api_call.py index 48a6be3..b73965c 100644 --- a/tests/test_taiga_api_call.py +++ b/tests/test_taiga_api_call.py @@ -1,8 +1,6 @@ """Tests for datasources/requests/taiga_api_call.py""" -import pytest from unittest.mock import patch, MagicMock -from datetime import datetime, timedelta class TestMilestoneStats: diff --git a/tests/test_taiga_auth.py b/tests/test_taiga_auth.py index e227255..6b24113 100644 --- a/tests/test_taiga_auth.py +++ b/tests/test_taiga_auth.py @@ -1,6 +1,7 @@ """Tests for utils/taiga_token/taiga_auth.py""" -import time, pytest +import time +import pytest from unittest.mock import patch, MagicMock diff --git a/tests/test_taiga_handler.py b/tests/test_taiga_handler.py index bcf0d86..d9a155c 100644 --- a/tests/test_taiga_handler.py +++ b/tests/test_taiga_handler.py @@ -1,7 +1,7 @@ """Tests for datasources/taiga_handler.py""" import pytest -from unittest.mock import patch, MagicMock +from unittest.mock import patch class TestParseTaigaEvent: @@ -240,10 +240,14 @@ def test_userstory_custom_attributes_none(self, mock_ms): assert result["priority"] == "" @patch("datasources.taiga_handler.milestone_stats", return_value={}) - @pytest.mark.xfail(reason="parse_taiga_userstory_event crashes when custom_attributes_values is None", strict=False) + @pytest.mark.xfail( + reason="parse_taiga_userstory_event crashes when custom_attributes_values is None", + strict=False, + ) def test_userstory_custom_attributes_values_none_xfail(self, mock_ms): """Explicitly exercise the None custom_attributes_values case to capture the known bug.""" from datasources.taiga_handler import parse_taiga_userstory_event + payload = { "type": "userstory", "action": "create", @@ -258,12 +262,14 @@ def test_userstory_custom_attributes_values_none_xfail(self, mock_ms): "description": "", "custom_attributes_values": None, "points": [{"value": None}, {"value": 3}], - "milestone": None + "milestone": None, }, - "is_closed": False + "is_closed": False, } with pytest.raises(AttributeError): parse_taiga_userstory_event(payload, "P") + + class TestParseTaigaRelatedUserstoryEvent: def test_basic_related_userstory_parsing(self, taiga_related_userstory_payload): from datasources.taiga_handler import parse_taiga_related_userstory_event diff --git a/tests/test_taiga_routes.py b/tests/test_taiga_routes.py index 228465f..0d69581 100644 --- a/tests/test_taiga_routes.py +++ b/tests/test_taiga_routes.py @@ -1,6 +1,8 @@ """Tests for routes/taiga_routes.py""" -import hashlib, hmac, json, pytest +import hashlib +import hmac +import json from unittest.mock import patch, MagicMock diff --git a/tests/test_verify_signature_github.py b/tests/test_verify_signature_github.py index 8c748e9..04f6cca 100644 --- a/tests/test_verify_signature_github.py +++ b/tests/test_verify_signature_github.py @@ -1,6 +1,7 @@ """Tests for routes/verify_signature/verify_signature_github.py""" -import hashlib, hmac, pytest +import hashlib +import hmac from unittest.mock import MagicMock from routes.verify_signature.verify_signature_github import verify_github_signature diff --git a/tests/test_verify_signature_taiga.py b/tests/test_verify_signature_taiga.py index 4affd32..a48bb81 100644 --- a/tests/test_verify_signature_taiga.py +++ b/tests/test_verify_signature_taiga.py @@ -1,6 +1,7 @@ """Tests for routes/verify_signature/verify_signature_taiga.py""" -import hashlib, hmac, pytest +import hashlib +import hmac from unittest.mock import MagicMock from routes.verify_signature.verify_signature_taiga import verify_taiga_signature diff --git a/utils/recovery/github_recovery.py b/utils/recovery/github_recovery.py index c67a3ae..1f2d8d4 100644 --- a/utils/recovery/github_recovery.py +++ b/utils/recovery/github_recovery.py @@ -1,4 +1,5 @@ -import argparse, requests +import argparse +import requests from typing import Dict, Iterable, Optional, List from datetime import datetime, timezone from pymongo import UpdateOne diff --git a/utils/recovery/taiga_recovery.py b/utils/recovery/taiga_recovery.py index 2bdf192..b63cb25 100644 --- a/utils/recovery/taiga_recovery.py +++ b/utils/recovery/taiga_recovery.py @@ -1,4 +1,6 @@ -import argparse, re, requests +import argparse +import re +import requests from pymongo import UpdateOne from datetime import datetime, timezone from typing import Optional, Dict, List @@ -6,11 +8,10 @@ import logging from database.mongo_client import get_collection -from utils.taiga_token.get_taiga_token import get_token from routes.API_publisher.API_event_publisher import notify_eval_push from config.logger_config import setup_logging -from config.settings import TAIGA_USERNAME, TAIGA_PASSWORD, TAIGA_API_URL +from config.settings import TAIGA_API_URL setup_logging() logger = logging.getLogger(__name__) diff --git a/utils/taiga_token/taiga_auth.py b/utils/taiga_token/taiga_auth.py index 0488c04..c75ac1d 100644 --- a/utils/taiga_token/taiga_auth.py +++ b/utils/taiga_token/taiga_auth.py @@ -1,4 +1,6 @@ -import requests, logging, time +import requests +import logging +import time log = logging.getLogger(__name__) _TOKENS = {} # key = (username, password) -> token diff --git a/utils/webhook_deletion/delete_webhooks_github.py b/utils/webhook_deletion/delete_webhooks_github.py index 52f9770..24ad7f2 100644 --- a/utils/webhook_deletion/delete_webhooks_github.py +++ b/utils/webhook_deletion/delete_webhooks_github.py @@ -5,7 +5,6 @@ MONGO_URI, MONGO_DB, GITHUB_TOKEN, - WEBHOOK_URL_GITHUB, GITHUB_API_URL, ) diff --git a/utils/webhook_deletion/delete_webhooks_taiga.py b/utils/webhook_deletion/delete_webhooks_taiga.py index 7e9cd41..0339a29 100644 --- a/utils/webhook_deletion/delete_webhooks_taiga.py +++ b/utils/webhook_deletion/delete_webhooks_taiga.py @@ -1,7 +1,7 @@ import logging import pymongo import requests -from config.settings import MONGO_URI, MONGO_DB, WEBHOOK_URL_TAIGA +from config.settings import MONGO_URI, MONGO_DB logger = logging.getLogger(__name__) REQUEST_TIMEOUT = 10 From b408e732705f37bd55395d476dd0894f0d671d83 Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Sun, 8 Mar 2026 13:01:35 +0100 Subject: [PATCH 12/32] ci: remove black, superseeded by ruff --- .github/workflows/linter.yml | 30 ------------------------------ 1 file changed, 30 deletions(-) delete mode 100644 .github/workflows/linter.yml diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml deleted file mode 100644 index b8fb285..0000000 --- a/.github/workflows/linter.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Linter - -on: - push: - branches: - - "**" - pull_request: - -jobs: - black: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.10" - cache: "pip" - cache-dependency-path: "requirements.txt" - - - name: Install Black - run: | - python -m pip install --upgrade pip - pip install black - - - name: Run Black - run: black --check . From 7a0376928bbe9253c34c645b8d900027986449d5 Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Sun, 8 Mar 2026 13:08:41 +0100 Subject: [PATCH 13/32] ci: removed tests workflow since is already at the ci pipeline --- .../{ci-security-pipeline.yml => ci.yml} | 12 ++---- .github/workflows/tests.yml | 40 ------------------- 2 files changed, 4 insertions(+), 48 deletions(-) rename .github/workflows/{ci-security-pipeline.yml => ci.yml} (90%) delete mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/ci-security-pipeline.yml b/.github/workflows/ci.yml similarity index 90% rename from .github/workflows/ci-security-pipeline.yml rename to .github/workflows/ci.yml index d9b1c8a..bd13958 100644 --- a/.github/workflows/ci-security-pipeline.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: CI Security Pipeline +name: CI on: push: @@ -12,10 +12,6 @@ permissions: jobs: test: runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.10", "3.11", "3.12"] steps: - name: Checkout repository @@ -24,7 +20,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: "3.14" cache: pip cache-dependency-path: requirements.txt @@ -47,7 +43,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.14" cache: pip - name: Install Ruff @@ -68,7 +64,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.14" cache: pip - name: Install Bandit diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index 98e84c7..0000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Tests - -on: - push: - branches: - - "**" - pull_request: - -jobs: - test: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.10" - cache: "pip" - cache-dependency-path: "requirements.txt" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install pytest pytest-mock pytest-cov - - - name: Run tests with coverage - run: pytest --cov=. --cov-report=term --cov-report=xml --cov-report=html - - - name: Upload coverage artifacts - uses: actions/upload-artifact@v4 - with: - name: coverage-report - path: | - coverage.xml - htmlcov/ - if-no-files-found: error From 415d6ea712caa4ed293b4987f06d3a6fda69926e Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Sun, 8 Mar 2026 13:36:23 +0100 Subject: [PATCH 14/32] refactor: Ruff checked! --- datasources/excel_handler.py | 2 -- datasources/taiga_handler.py | 6 +++--- routes/excel_routes.py | 3 ++- routes/github_routes.py | 1 + routes/taiga_routes.py | 12 +++++++----- tests/test_app.py | 1 - 6 files changed, 13 insertions(+), 12 deletions(-) diff --git a/datasources/excel_handler.py b/datasources/excel_handler.py index f422c2d..153034c 100644 --- a/datasources/excel_handler.py +++ b/datasources/excel_handler.py @@ -1,5 +1,3 @@ - - # List of activity types extracted from the Excel sheet ACTIVITY_TYPES = [ "Reunió d'equip", diff --git a/datasources/taiga_handler.py b/datasources/taiga_handler.py index 62858e0..853983d 100644 --- a/datasources/taiga_handler.py +++ b/datasources/taiga_handler.py @@ -53,7 +53,7 @@ def parse_taiga_issue_event(raw_payload: Dict, prj: str) -> Dict: ) assigned_by = raw_payload.get("by", {}).get("username", "") # There are cases where the assigned_to field is empty, and if we request it aniways it will throw an error, so we need to check if it exists - if raw_payload.get("data", {}).get("assigned_to", {}) != None: + if raw_payload.get("data", {}).get("assigned_to", {}) is not None: assigned_to = ( raw_payload.get("data", {}).get("assigned_to", {}).get("username", "") ) @@ -180,7 +180,7 @@ def parse_taiga_task_event(raw_payload: Dict, prj: str) -> Dict: custom_attributes = {} # There are cases where the assigned_to field is empty, and if we request it aniways it will throw an error, so we need to check if it exists - if raw_payload.get("data", {}).get("assigned_to", {}) != None: + if raw_payload.get("data", {}).get("assigned_to", {}) is not None: assigned_to = ( raw_payload.get("data", {}).get("assigned_to", {}).get("username", "") ) @@ -254,7 +254,7 @@ def parse_taiga_userstory_event(raw_payload: Dict, prj: str) -> Dict: pattern_in_description = False # If the userstory has a milestone associated while created, we will get the values, if not, we will set them to None - if raw_payload.get("data", {}).get("milestone", {}) != None: + if raw_payload.get("data", {}).get("milestone", {}) is not None: milestone_id = raw_payload.get("data", {}).get("milestone", {}).get("id", "") milestone_name = ( diff --git a/routes/excel_routes.py b/routes/excel_routes.py index 8033dbc..e59b065 100644 --- a/routes/excel_routes.py +++ b/routes/excel_routes.py @@ -41,7 +41,8 @@ def excel_webhook(): # Create the collection name based on the project ID collection_name = f"{prj}_sheets" event_name = "sheets_activity" - author_login = "" # username of the author + # author_login = "" # username of the author + logger.info(f"Processing Excel event: {event_name} for team: {prj} with quality_model: {quality_model}") coll = get_collection(collection_name) diff --git a/routes/github_routes.py b/routes/github_routes.py index 4bf97ae..e480ec9 100644 --- a/routes/github_routes.py +++ b/routes/github_routes.py @@ -57,6 +57,7 @@ def github_webhook(): team_name = parsed_data[ "team_name" ] # We wont use this, we will use the external_id instead as its a CENTRALIZED ID + logger.info(f"Processing Github event: {parsed_data['event']} for team: {team_name} (external_id: {prj})") event_label = parsed_data["event"] # This is either "commit" or "issue" author_login = parsed_data["sender_info"][ "login" diff --git a/routes/taiga_routes.py b/routes/taiga_routes.py index 17ed231..f748714 100644 --- a/routes/taiga_routes.py +++ b/routes/taiga_routes.py @@ -39,6 +39,7 @@ def taiga_webhook(): action_type = raw_payload.get("action", "") id = raw_payload.get("data", {}).get("id", "") team_name = raw_payload.get("data", {}).get("project", {}).get("name", "") + logger.info(f"Processing Taiga event: {event_type} with action: {action_type} for team: {team_name} (external_id: {prj})") # Decide the Mongo collection name to write to, depending on the event type if event_type in ["userstory", "relateduserstory"]: @@ -98,8 +99,9 @@ def taiga_webhook(): logger.info(f"Upserting task with ID: {task_id}") # Upsert instead of insert - parsed_data["prj"] = prj - result = coll.update_one( + result = parsed_data["prj"] = prj + logger.info(f"Upserting task with ID: {task_id} for team {prj} in collection {collection_name} result: {result}") + coll.update_one( {"task_id": task_id}, {"$set": parsed_data}, upsert=True ) logger.info(f"Inserting in MongoDB Taiga task for team {prj}") @@ -114,7 +116,7 @@ def taiga_webhook(): logger.info(f"Upserting epic with ID: {epic_id}") # Upsert instead of insert parsed_data["prj"] = prj - result = coll.update_one( + coll.update_one( {"epic_id": epic_id}, {"$set": parsed_data}, upsert=True ) logger.info(f"Inserting in MongoDB Taiga epic for team {prj}") @@ -129,7 +131,7 @@ def taiga_webhook(): logger.info(f"Upserting issue with ID: {issue_id}") parsed_data["prj"] = prj # Upsert instead of insert - result = coll.update_one( + coll.update_one( {"issue_id": issue_id}, {"$set": parsed_data}, upsert=True ) logger.info(f"Inserting in MongoDB Taiga issue for team {prj}") @@ -137,7 +139,7 @@ def taiga_webhook(): else: # If the event is not one of the above, we will insert it as a new document parsed_data["prj"] = prj - inserted_id = coll.insert_one(parsed_data).inserted_id + coll.insert_one(parsed_data) # COMMUNICATION WITH LD_EVAL USING API logger.info( diff --git a/tests/test_app.py b/tests/test_app.py index cf82a91..b01b7e3 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -1,7 +1,6 @@ """Tests for app.py""" - class TestCreateApp: def test_app_created(self, flask_app): assert flask_app is not None From 12c4c0caadbd87d28dfced939280d0893b456e2d Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Sun, 8 Mar 2026 13:36:38 +0100 Subject: [PATCH 15/32] ci: excluded deprecated folder and tests --- .github/workflows/ci.yml | 4 ++-- pyproject.toml | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd13958..f7d1366 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: pip install ruff - name: Run Ruff - run: ruff check . + run: ruff check . --exclude tests,deprecated bandit: runs-on: ubuntu-latest @@ -73,7 +73,7 @@ jobs: pip install bandit - name: Run Bandit - run: bandit -r . -x tests -ll + run: bandit -r . -x tests -ll -x tests,deprecated gitleaks: runs-on: ubuntu-latest diff --git a/pyproject.toml b/pyproject.toml index cc44f6f..5fad333 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,3 +19,6 @@ omit = [ [tool.coverage.report] show_missing = true fail_under = 80 + +[tool.ruff] +extend-exclude = ["deprecated"] From 6a86ea57fba2890faa68e4060655dd2663a10dd2 Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Sun, 8 Mar 2026 19:47:07 +0100 Subject: [PATCH 16/32] ci: added coverage artifact --- .github/workflows/ci.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7d1366..12f853b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,8 +30,17 @@ jobs: pip install -r requirements.txt pip install pytest - - name: Run tests - run: pytest + - name: Run tests with coverage + run: pytest --cov=. --cov-report=term --cov-report=xml --cov-report=html + + - name: Upload coverage artifacts + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: | + coverage.xml + htmlcov/ + if-no-files-found: error lint: runs-on: ubuntu-latest From fb9993297acc118b42a94c5920a4628406112fff Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Sun, 8 Mar 2026 19:47:28 +0100 Subject: [PATCH 17/32] fix: changed old requirements to bump into 3.14 --- requirements.old.txt | Bin 0 -> 1296 bytes requirements.txt | Bin 1296 -> 1324 bytes template.env | 14 ++++---------- 3 files changed, 4 insertions(+), 10 deletions(-) create mode 100644 requirements.old.txt diff --git a/requirements.old.txt b/requirements.old.txt new file mode 100644 index 0000000000000000000000000000000000000000..b369a4c0676065c7a2c81f53f8f0fb3e278829e4 GIT binary patch literal 1296 zcmbu9%TB^j5Qb-M;-g@scwM-4VdBb#OH!eLw^mzfz=v1=Z>EQXZcS4lXXgBux$yl} zTCJzCwUv4+o7+g=TYJ;*&aU(v+mW5?J+?zTP*$zzqizs8(fQ>r_FylH)b^~qt;ff| z;xj*AD`%mX`-M;oJ5?5$6Me!lvz2&S@mJC^a(%76>J0qP9r{FfP>^mBorxXfivjau zKx}lk5oYN)$rrcJJYz1FPOi6NTq(02^zD0}dz~u9K0HcgaDV9tm2Jdc2?K{9EXCJY zuM_xiSon*_g_Ly4EESDffHu|>)d)}5QcYfX!Y?S%iKUQhcZ{#3mmH8HXXS{=?bxdo zkw#Sx40hJIfZ+O+jJ|GQt{akr$#VDU;pMzl?Pv_2}f{dL^Q#cC!DS@ zo%C&gLPpNc*7V-&%#GIfFW)&d99|44MPR^RP97{OIh(h5VfBwXEW%9ml}nt*uXs5Yi%f9 wfgx0<9giCK&daVo$W8W@=Bn3r(ak*7=$h=!?!B}yTl^FB!(61@_+OWP01c|WMgRZ+ literal 0 HcmV?d00001 diff --git a/requirements.txt b/requirements.txt index b369a4c0676065c7a2c81f53f8f0fb3e278829e4..999ed9e0c22d9cda9aa61ddbb2499898bdfbbe38 100644 GIT binary patch delta 277 zcmbQhwT5eh9^>RZW+`4f23rP020aD?5Z*kSQJHZv4|CGwSh}V~A%c a1DaU`baN({ufSjfRH4LR40ewpkOcq}g)Gwm delta 222 zcmZ3(HGyk`9;0LmLq0)waCb9qk diff --git a/template.env b/template.env index 0f4440c..7b8e16d 100644 --- a/template.env +++ b/template.env @@ -13,11 +13,9 @@ MONGO_USER= MONGO_PASS= MONGO_AUTHSRC= +CREDENTIALS_FILE=config_files/credentials_config.json -//#The github token should be the token of an administrator of all the repositories/organizations of all the students -//Also when creating this token we need to select 'admin:repo_hook' and also 'repo' to have full access to the commits stats - - +#### GitHub configuration #This is the secret key that we will use to verify the signature. I must be the same as the one in the webhook GITHUB_SIGNATURE_KEY="myGHS3cr3t!" @@ -25,12 +23,6 @@ GITHUB_SIGNATURE_KEY="myGHS3cr3t!" # GitHub API URL (default: https://api.github.com, change for GitHub Enterprise) GITHUB_API_URL=https://api.github.com - - -#This is the secret key that we will use to verify the signature. I must be the same as the one in the webhook -TAIGA_SIGNATURE_KEY="asdqasfs" - - #### Configuración de Taiga # URL de la API de Taiga @@ -41,6 +33,8 @@ TAIGA_API_URL=https://api.taiga.io/api/v1 # URL de autenticación de Taiga (defaults to TAIGA_API_URL if not set) TAIGA_AUTH_URL=https://api.taiga.io/api/v1 +#This is the secret key that we will use to verify the signature. I must be the same as the one in the webhook +TAIGA_SIGNATURE_KEY="asdqasfs" # URLS de los webhooks WEBHOOK_URL_GITHUB="https://gessi-dashboard.essi.upc.edu/webhook/github" From f5b54908f375dd4094b3c0a619341805af7b31e7 Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Sun, 8 Mar 2026 20:55:01 +0100 Subject: [PATCH 18/32] fix: removed unused dependencies --- requirements.txt | Bin 1324 -> 402 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/requirements.txt b/requirements.txt index 999ed9e0c22d9cda9aa61ddbb2499898bdfbbe38..8c0dc96abec581c60142f6a03e25a26b0ec0d787 100644 GIT binary patch literal 402 zcmXYty;8(541{~X%Xu982%&IG!3-4zwJ*`-OdKot2e>>v>p*8~SJJ1w1Robl)^_qq zDlt+j{_3sONnof*2mok-81+f=X%-g-( zIAWH3#OUYXGTa7kCFUCa$f=1&Dir(_(FzgIk|!5uN?dbm){tL({BZiOyBDu6Ep>Zy zS85d{wOG0E!PPcxZd}~#MiW#XF^CrwY6l Oh7sR4B2XZ=9=~679Cz&i literal 1324 zcmbu9OHbTT428X})c=6GA4D?HQuVQ9fy9ahOBCjr7LsWm2>9{P@7TE{gHShB6o{_v zC!UwBOSM7Rs+B3?9=EBrKH-5%zOJuwkD8Pq0<@t z3J`;hm_}UlZLFtCT*WALQqEYvB|mcxI?H^{KrZQAfvp(=H=TF%TN0`0pbgaYi5=!( z=#9*CdRO*~%zT29l(^u$fp?D@mwd0mO1;D-cfIa>dht)O(IY#mw1)y7>a_8k+-Hbt zsUS6lqWI`0O)V#+jI<5u9S-eLy+mF0)lA{PL-1;*?S>f}SpQ)LxmdcS?77zjehbcn zN7OT$xV(jxTq>Uiw_ugL7B?hjyQas(-~=IRqShS|-W(m;;oX=bwl}&zROwOkmZ!8E z6c`z&z4bV4-!uHlbgc9V}xL8}!gU*g3Yg7nL N_MMDR!wt*$`U@Qr!Vmxe From 2c4ff3ca973ad7bc14c1cfbc944057a296ff0ab4 Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Sun, 8 Mar 2026 20:55:08 +0100 Subject: [PATCH 19/32] fix: corrected casing and formatting in docker-compose.yml --- docker-compose.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 143bb4f..945a5bf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,8 @@ services: - LD_connect: + ld_connect: build: context: . - dockerfile: Dockerfile + dockerfile: dockerfile container_name: LDConnect environment: - EVAL_HOST=ld_eval @@ -10,10 +10,10 @@ services: env_file: - .env networks: - - qrapids + - qrapids depends_on: - mongodb: - condition: service_started + mongodb: + condition: service_started ports: - "127.0.0.1:5000:5000" From c6ce3a44854dfdf151ea727ae615fe1f765bf9ed Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Sun, 8 Mar 2026 20:55:12 +0100 Subject: [PATCH 20/32] ci: update test dependencies to include pytest-cov for coverage reporting --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12f853b..bbb3de8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt - pip install pytest + pip install pytest pytest-cov - name: Run tests with coverage run: pytest --cov=. --cov-report=term --cov-report=xml --cov-report=html From 97000408916cbe7ce2b63d9fdf82a05b4e1370b4 Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Sun, 8 Mar 2026 21:03:39 +0100 Subject: [PATCH 21/32] ci: add workflow to restrict PRs to develop branch only --- .github/workflows/main-pr.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/main-pr.yml diff --git a/.github/workflows/main-pr.yml b/.github/workflows/main-pr.yml new file mode 100644 index 0000000..f79bfe1 --- /dev/null +++ b/.github/workflows/main-pr.yml @@ -0,0 +1,17 @@ +name: Restrict PR Source + +on: + pull_request: + branches: + - main + +jobs: + check-branch: + runs-on: ubuntu-latest + steps: + - name: Fail if not from develop + run: | + if [[ "${{ github.head_ref }}" != "develop" ]]; then + echo "PR must come from develop branch only." + exit 1 + fi \ No newline at end of file From 9da7977f542b2a04076a590b4bfcb54d3a24ea41 Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Sun, 8 Mar 2026 21:51:10 +0100 Subject: [PATCH 22/32] ci: add pre-commit configuration for code quality checks --- .pre-commit-config.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..2c3b683 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.11.0 + hooks: + - id: ruff + args: ["--fix"] + + - repo: https://github.com/gitleaks/gitleaks + rev: v8.30.0 + hooks: + - id: gitleaks + args: ["--redact", "--verbose", "--baseline-path", ".gitleaks-baseline.json", "--exit-code", "1"] From 9a059e76b7e38a9d94ecf5ea280b9979eaf5e3a8 Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Sun, 8 Mar 2026 21:51:41 +0100 Subject: [PATCH 23/32] style: removed trailing whitespaces --- .github/workflows/main-pr.yml | 2 +- .gitignore | 4 +-- README.md | 58 +++++++++++++++++++++++++++------- docker-compose.yml | 14 ++++---- requirements.old.txt | Bin 1296 -> 1297 bytes template.env | 6 ++-- 6 files changed, 60 insertions(+), 24 deletions(-) diff --git a/.github/workflows/main-pr.yml b/.github/workflows/main-pr.yml index f79bfe1..6386999 100644 --- a/.github/workflows/main-pr.yml +++ b/.github/workflows/main-pr.yml @@ -14,4 +14,4 @@ jobs: if [[ "${{ github.head_ref }}" != "develop" ]]; then echo "PR must come from develop branch only." exit 1 - fi \ No newline at end of file + fi diff --git a/.gitignore b/.gitignore index 75dd118..6056d80 100644 --- a/.gitignore +++ b/.gitignore @@ -195,9 +195,9 @@ cython_debug/ .abstra/ # Visual Studio Code -# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore # that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore -# and can be added to the global gitignore or merged into this file. However, if you prefer, +# and can be added to the global gitignore or merged into this file. However, if you prefer, # you could uncomment the following to ignore the entire vscode folder # .vscode/ diff --git a/README.md b/README.md index c11df9c..48b77e0 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ # LD Connect – Event Ingestion Service -**LD Connect** is the entry point of the Learning Dashboard pipeline. +**LD Connect** is the entry point of the Learning Dashboard pipeline. Whenever a student pushes to **GitHub**, edits a task on **Taiga**, or logs effort in **Google Sheets**, the event first reaches this service. LD Connect -1. **Authenticates** the webhook (HMAC signatures) -2. **Normalises** the payload to a common schema -3. **Persists** it in MongoDB (idempotent upserts) +1. **Authenticates** the webhook (HMAC signatures) +2. **Normalises** the payload to a common schema +3. **Persists** it in MongoDB (idempotent upserts) 4. **Notifies** LD Eval so metrics are recalculated in near real‑time --- @@ -27,10 +27,10 @@ Whenever a student pushes to **GitHub**, edits a task on **Taiga**, or logs effo ```text ┌──────────────┐ Webhook ┌──────────────┐ │ GitHub │───POST────▶ │ │ -└──────────────┘ │ │ -┌──────────────┐ │ │ +└──────────────┘ │ │ +┌──────────────┐ │ │ │ Taiga │───POST────▶ │ LD Connect │──┐ POST /api/event -└──────────────┘ │ │ │ +└──────────────┘ │ │ │ ┌──────────────┐ │ │ │ │ GoogleSheet │───POST────▶│ │ │ (notify) └──────────────┘ └──────────────┘ │ @@ -90,8 +90,8 @@ curl -X POST "http://127.0.0.1:5000/webhook/github?ping=1" docker compose up -d --build ld_connect ``` -* Exposes the service on port **5000** inside the container -* Behind Nginx / Traefik, route +* Exposes the service on port **5000** inside the container +* Behind Nginx / Traefik, route `https:///webhook/{github|taiga|excel}` → `ld_connect:5000` --- @@ -115,12 +115,12 @@ Store them in `.env` (already referenced in `docker-compose.yml`). ### `POST /webhook/github` -Receives any GitHub event subscribed in the repo webhook. +Receives any GitHub event subscribed in the repo webhook. Requires headers `X-Hub-Signature` **and** `X-Hub-Signature-256`. ### `POST /webhook/taiga` -Receives Taiga events. +Receives Taiga events. Requires header `X-Taiga-Webhook-Signature`. ### `POST /webhook/excel` @@ -144,6 +144,42 @@ All endpoints return **`200 OK`** immediately; heavy work continues asynchronou pytest # unit tests ``` +If you just cloned the repository, set up `pre-commit` once before you start coding. +It will automatically run checks every time you commit, so common issues are caught early. + +### First-time setup (after cloning) + +```bash +# 1) create and activate a virtual environment (if you did not do it yet) +python -m venv .venv +source .venv/bin/activate + +# 2) install project dependencies +pip install -r requirements.txt + +# 3) install pre-commit in your environment +pip install pre-commit + +# 4) install git hooks for this repository (one-time) +pre-commit install + +# 5) optional: run checks on all files now +pre-commit run --all-files +``` + +### Why this helps + +- `ruff` catches Python style/quality issues and can auto-fix many of them. +- `gitleaks` helps prevent committing secrets (tokens, passwords, keys). +- Because hooks run before each commit, problems are found locally instead of failing later in CI. + +Configured hooks: + +| Hook | Purpose | +| --- | --- | +| `ruff` | Python linting (with autofix) | +| `gitleaks` | Detect hardcoded secrets | + --- ## License diff --git a/docker-compose.yml b/docker-compose.yml index 945a5bf..738585c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,23 +1,23 @@ services: ld_connect: build: - context: . + context: . dockerfile: dockerfile container_name: LDConnect environment: - EVAL_HOST=ld_eval - EVAL_PORT=5001 env_file: - - .env + - .env networks: - - qrapids + - qrapids depends_on: - mongodb: - condition: service_started + mongodb: + condition: service_started ports: - - "127.0.0.1:5000:5000" + - "127.0.0.1:5000:5000" networks: qrapids: - external: true + external: true diff --git a/requirements.old.txt b/requirements.old.txt index b369a4c0676065c7a2c81f53f8f0fb3e278829e4..e38aac5351fb35f68fcecd24f30bc4aa5f42594b 100644 GIT binary patch delta 9 QcmbQhHIZwB04pOG01amX1^@s6 delta 7 OcmbQpHGyk`04o3qX94>F diff --git a/template.env b/template.env index 7b8e16d..c4fffb5 100644 --- a/template.env +++ b/template.env @@ -1,5 +1,5 @@ # Set to True to enable debug mode, which will print more detailed logs and error messages. Set to False for production use. -DEBUG=False +DEBUG=False # Flask server settings (only used when running python app.py directly) FLASK_DEBUG=false @@ -18,7 +18,7 @@ CREDENTIALS_FILE=config_files/credentials_config.json #### GitHub configuration #This is the secret key that we will use to verify the signature. I must be the same as the one in the webhook -GITHUB_SIGNATURE_KEY="myGHS3cr3t!" +GITHUB_SIGNATURE_KEY="myGHS3cr3t!" # GitHub API URL (default: https://api.github.com, change for GitHub Enterprise) GITHUB_API_URL=https://api.github.com @@ -34,7 +34,7 @@ TAIGA_API_URL=https://api.taiga.io/api/v1 TAIGA_AUTH_URL=https://api.taiga.io/api/v1 #This is the secret key that we will use to verify the signature. I must be the same as the one in the webhook -TAIGA_SIGNATURE_KEY="asdqasfs" +TAIGA_SIGNATURE_KEY="asdqasfs" # URLS de los webhooks WEBHOOK_URL_GITHUB="https://gessi-dashboard.essi.upc.edu/webhook/github" From 59afe4b7fd4e3259b6af8af96be4b1f2a108bf06 Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Sun, 8 Mar 2026 22:03:52 +0100 Subject: [PATCH 24/32] ci: simplified precommit hooks --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2c3b683..960cd8e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,4 +15,4 @@ repos: rev: v8.30.0 hooks: - id: gitleaks - args: ["--redact", "--verbose", "--baseline-path", ".gitleaks-baseline.json", "--exit-code", "1"] + args: ["--redact=0", "--baseline-path", ".gitleaks-baseline.json", "--exit-code", "1"] From 5397e3bf18e0ecceea9ba1f7e983b3ff6d9bef5c Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Fri, 13 Mar 2026 12:51:25 +0100 Subject: [PATCH 25/32] added new branch dev to git flow --- .github/workflows/main-pr.yml | 2 +- .../requirements.old.txt | Bin docker-compose.yml | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename requirements.old.txt => deprecated/requirements.old.txt (100%) diff --git a/.github/workflows/main-pr.yml b/.github/workflows/main-pr.yml index 6386999..50213e6 100644 --- a/.github/workflows/main-pr.yml +++ b/.github/workflows/main-pr.yml @@ -11,7 +11,7 @@ jobs: steps: - name: Fail if not from develop run: | - if [[ "${{ github.head_ref }}" != "develop" ]]; then + if [[ "${{ github.head_ref }}" != "dev" ]]; then echo "PR must come from develop branch only." exit 1 fi diff --git a/requirements.old.txt b/deprecated/requirements.old.txt similarity index 100% rename from requirements.old.txt rename to deprecated/requirements.old.txt diff --git a/docker-compose.yml b/docker-compose.yml index 738585c..11bbf69 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,7 @@ services: networks: - qrapids depends_on: - mongodb: + mongodb: condition: service_started ports: - "127.0.0.1:5000:5000" From afe91149bb53244aae02d78450491bfcb0d3b114 Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Fri, 13 Mar 2026 13:11:35 +0100 Subject: [PATCH 26/32] fix: added exclusions to sonarcloud --- .sonarcloud.properties | 1 + 1 file changed, 1 insertion(+) create mode 100644 .sonarcloud.properties diff --git a/.sonarcloud.properties b/.sonarcloud.properties new file mode 100644 index 0000000..ec7d0d1 --- /dev/null +++ b/.sonarcloud.properties @@ -0,0 +1 @@ +sonar.exclusions=deprecated/**,.gitleaks-baseline.json From 2581f9d3fee190f622ac144cddcb30c50c6f5464 Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Fri, 13 Mar 2026 13:18:28 +0100 Subject: [PATCH 27/32] fixed some sonarcube errors --- .github/workflows/ci.yml | 14 +++++++++++--- .github/workflows/main-pr.yml | 5 ++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bbb3de8..2357fff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,11 +6,12 @@ on: pull_request: branches: [main] -permissions: - contents: read + jobs: test: + permissions: + contents: read runs-on: ubuntu-latest steps: @@ -43,6 +44,8 @@ jobs: if-no-files-found: error lint: + permissions: + contents: read runs-on: ubuntu-latest steps: @@ -64,6 +67,8 @@ jobs: run: ruff check . --exclude tests,deprecated bandit: + permissions: + contents: read runs-on: ubuntu-latest steps: @@ -86,6 +91,8 @@ jobs: gitleaks: runs-on: ubuntu-latest + permissions: + contents: read steps: - name: Checkout repository (full history) @@ -105,7 +112,8 @@ jobs: semgrep: runs-on: ubuntu-latest - + permissions: + contents: read steps: - name: Checkout repository uses: actions/checkout@v4 diff --git a/.github/workflows/main-pr.yml b/.github/workflows/main-pr.yml index 50213e6..bc61ee5 100644 --- a/.github/workflows/main-pr.yml +++ b/.github/workflows/main-pr.yml @@ -8,10 +8,13 @@ on: jobs: check-branch: runs-on: ubuntu-latest + env: + GITHUB_HEAD_REF: ${{ github.head_ref }} steps: + - name: Fail if not from develop run: | - if [[ "${{ github.head_ref }}" != "dev" ]]; then + if [[ $GITHUB_HEAD_REF != "dev" ]]; then echo "PR must come from develop branch only." exit 1 fi From 33856bbc8ca5d05f7c1dce851568506968da0a24 Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Fri, 13 Mar 2026 13:56:02 +0100 Subject: [PATCH 28/32] fix: add tests directory to sonar exclusions --- .sonarcloud.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.sonarcloud.properties b/.sonarcloud.properties index ec7d0d1..8c0335f 100644 --- a/.sonarcloud.properties +++ b/.sonarcloud.properties @@ -1 +1 @@ -sonar.exclusions=deprecated/**,.gitleaks-baseline.json +sonar.exclusions=deprecated/**,.gitleaks-baseline.json,tests/**,deprecated/** From 06e3e9054fbaebee0426e3699d3a1d4099ecce57 Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Fri, 13 Mar 2026 14:01:31 +0100 Subject: [PATCH 29/32] ci: add checks also to dev --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2357fff..403bd7e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: push: branches: [main] pull_request: - branches: [main] + branches: [main, dev] From dfa2c5760ff564ff2cb7d742eabf5235fc3226e4 Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Sat, 14 Mar 2026 13:50:52 +0100 Subject: [PATCH 30/32] refactor: simplify function signatures and improve response handling in webhooks --- datasources/github_handler.py | 4 ++-- datasources/taiga_handler.py | 6 +----- routes/excel_routes.py | 6 +++--- routes/github_routes.py | 10 +++++----- tests/test_github_handler.py | 6 +++--- tests/test_taiga_handler.py | 16 +++++++--------- 6 files changed, 21 insertions(+), 27 deletions(-) diff --git a/datasources/github_handler.py b/datasources/github_handler.py index e92b281..72b35f6 100644 --- a/datasources/github_handler.py +++ b/datasources/github_handler.py @@ -14,7 +14,7 @@ def parse_github_event(raw_payload: Dict, prj: str) -> Dict: if event_type == "push": return parse_github_push_event(raw_payload, prj) elif event_type == "issues": - return parse_github_issue_event(raw_payload, prj) + return parse_github_issue_event(raw_payload) elif event_type == "pull_request": return parse_github_pullrequest_event(raw_payload, prj) else: @@ -112,7 +112,7 @@ def parse_github_push_event(raw_payload: Dict, prj: str) -> Dict: } -def parse_github_issue_event(raw_payload: Dict, prj: str) -> Dict: +def parse_github_issue_event(raw_payload: Dict) -> Dict: """ Function to parse a GitHub issue event payload. """ diff --git a/datasources/taiga_handler.py b/datasources/taiga_handler.py index 853983d..d01c0bf 100644 --- a/datasources/taiga_handler.py +++ b/datasources/taiga_handler.py @@ -291,11 +291,7 @@ def parse_taiga_userstory_event(raw_payload: Dict, prj: str) -> Dict: estimated_finish = "" milestone_data = {} - priority = ( - raw_payload.get("data", {}) - .get("custom_attributes_values", {}) - .get("Priority", "") - ) + priority = custom_attributes.get("Priority", "") points_list = raw_payload.get("data", {}).get("points", []) sum_points = sum(p.get("value") or 0 for p in points_list) diff --git a/routes/excel_routes.py b/routes/excel_routes.py index e59b065..ca10c61 100644 --- a/routes/excel_routes.py +++ b/routes/excel_routes.py @@ -1,4 +1,4 @@ -from flask import Blueprint, request, jsonify +from flask import Blueprint, make_response, request, jsonify from datasources.excel_handler import parse_excel_event from database.mongo_client import get_collection from config.logger_config import setup_logging @@ -20,7 +20,7 @@ def excel_webhook(): raw_json = request.get_json() if not raw_json: logger.warning("Excel webhook called without JSON payload.") - return {"error": "No JSON received"}, 400 + return make_response(jsonify({"error": "No JSON received"}), 400) # Read the query parameters from the request prj = request.args.get("prj", type=str) @@ -35,7 +35,7 @@ def excel_webhook(): # Parse the raw JSON payload using the parse_excel_event function parsed_data = parse_excel_event(raw_json, prj, quality_model) if "error" in parsed_data: - return parsed_data, 400 + return make_response(jsonify(parsed_data), 400) logger.info("Excel webhook request processed successfully.") # Create the collection name based on the project ID diff --git a/routes/github_routes.py b/routes/github_routes.py index e480ec9..293702a 100644 --- a/routes/github_routes.py +++ b/routes/github_routes.py @@ -1,4 +1,4 @@ -from flask import Blueprint, request, jsonify +from flask import Blueprint, make_response, request, jsonify from datasources.github_handler import parse_github_event from database.mongo_client import get_collection from config.settings import GITHUB_SIGNATURE_KEY @@ -50,9 +50,9 @@ def github_webhook(): logger.info(f"Github webhook request processed successfully for team {prj}.") if parsed_data.get("ignored"): - return {"status": "ignored", "event": parsed_data["event"]}, 200 + return make_response(jsonify({"status": "ignored", "event": parsed_data["event"]}), 200) if "error" in parsed_data: - return parsed_data, 400 + return make_response(jsonify(parsed_data), 400) team_name = parsed_data[ "team_name" @@ -97,14 +97,14 @@ def github_webhook(): coll.insert_one(commit_doc) logger.info(f"Inserting in MongoDB Github commit for team {prj}") - return {"status": "ok", "message": "Commits inserted"}, 200 + return make_response(jsonify({"status": "ok", "message": "Commits inserted"}), 200) # If it's an issue event, we insert the issue document elif "issue" in parsed_data: parsed_data["prj"] = prj coll.insert_one(parsed_data) logger.info(f"Inserting in MongoDB Github issue for team {prj}") - return {"status": "ok", "message": "Issue inserted"}, 200 + return make_response(jsonify({"status": "ok", "message": "Issue inserted"}), 200) elif "pull_request" in parsed_data: parsed_data["prj"] = prj diff --git a/tests/test_github_handler.py b/tests/test_github_handler.py index b1c7ff8..d0e681a 100644 --- a/tests/test_github_handler.py +++ b/tests/test_github_handler.py @@ -209,7 +209,7 @@ class TestParseGithubIssueEvent: def test_basic_issue_parsing(self, github_issue_payload): from datasources.github_handler import parse_github_issue_event - result = parse_github_issue_event(github_issue_payload, "TestPrj") + result = parse_github_issue_event(github_issue_payload) assert result["event"] == "issue" assert result["action"] == "opened" @@ -219,7 +219,7 @@ def test_basic_issue_parsing(self, github_issue_payload): def test_issue_object(self, github_issue_payload): from datasources.github_handler import parse_github_issue_event - result = parse_github_issue_event(github_issue_payload, "TestPrj") + result = parse_github_issue_event(github_issue_payload) issue = result["issue"] assert issue["number"] == 10 @@ -231,7 +231,7 @@ def test_issue_object(self, github_issue_payload): def test_sender_info(self, github_issue_payload): from datasources.github_handler import parse_github_issue_event - result = parse_github_issue_event(github_issue_payload, "TestPrj") + result = parse_github_issue_event(github_issue_payload) assert result["sender_info"]["login"] == "issueuser" assert result["sender_info"]["id"] == 2 diff --git a/tests/test_taiga_handler.py b/tests/test_taiga_handler.py index d9a155c..bbea108 100644 --- a/tests/test_taiga_handler.py +++ b/tests/test_taiga_handler.py @@ -1,6 +1,5 @@ """Tests for datasources/taiga_handler.py""" -import pytest from unittest.mock import patch @@ -240,12 +239,8 @@ def test_userstory_custom_attributes_none(self, mock_ms): assert result["priority"] == "" @patch("datasources.taiga_handler.milestone_stats", return_value={}) - @pytest.mark.xfail( - reason="parse_taiga_userstory_event crashes when custom_attributes_values is None", - strict=False, - ) - def test_userstory_custom_attributes_values_none_xfail(self, mock_ms): - """Explicitly exercise the None custom_attributes_values case to capture the known bug.""" + def test_userstory_custom_attributes_values_none(self, mock_ms): + """Ensure None custom_attributes_values is handled without crashing.""" from datasources.taiga_handler import parse_taiga_userstory_event payload = { @@ -266,8 +261,11 @@ def test_userstory_custom_attributes_values_none_xfail(self, mock_ms): }, "is_closed": False, } - with pytest.raises(AttributeError): - parse_taiga_userstory_event(payload, "P") + result = parse_taiga_userstory_event(payload, "P") + + assert result["custom_attributes"] == {} + assert result["priority"] == "" + assert result["total_points"] == 3 class TestParseTaigaRelatedUserstoryEvent: From bad887b58f3fe42345540a977f82a90e2e4aeacb Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Sat, 14 Mar 2026 14:02:06 +0100 Subject: [PATCH 31/32] refactor: update datetime handling and simplify function signatures in taiga API calls --- datasources/requests/taiga_api_call.py | 4 ++-- datasources/taiga_handler.py | 4 ++-- tests/test_taiga_handler.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/datasources/requests/taiga_api_call.py b/datasources/requests/taiga_api_call.py index 31ed12c..3e65850 100644 --- a/datasources/requests/taiga_api_call.py +++ b/datasources/requests/taiga_api_call.py @@ -1,6 +1,6 @@ import logging import requests -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from utils.taiga_token.taiga_auth import get_taiga_token from config.credentials_loader import resolve @@ -24,7 +24,7 @@ def milestone_stats(project_id: str, milestone_id: str, prj: str): return {} key = (project_id, milestone_id) - now = datetime.utcnow() + now = datetime.now(timezone.utc) if key in _CACHE and now - _CACHE[key][0] < TTL: return _CACHE[key][1] diff --git a/datasources/taiga_handler.py b/datasources/taiga_handler.py index d01c0bf..486cd16 100644 --- a/datasources/taiga_handler.py +++ b/datasources/taiga_handler.py @@ -13,7 +13,7 @@ def parse_taiga_event(raw_payload: Dict, prj: str) -> Dict: """ event_type = raw_payload.get("type") if event_type == "issue": - return parse_taiga_issue_event(raw_payload, prj) + return parse_taiga_issue_event(raw_payload) elif event_type == "epic": return parse_taiga_epic_event(raw_payload, prj) elif event_type == "task": @@ -26,7 +26,7 @@ def parse_taiga_event(raw_payload: Dict, prj: str) -> Dict: return {"event": event_type, "error": "Unsupported event type"} -def parse_taiga_issue_event(raw_payload: Dict, prj: str) -> Dict: +def parse_taiga_issue_event(raw_payload: Dict) -> Dict: """ Function to parse a taiga issue event payload. """ diff --git a/tests/test_taiga_handler.py b/tests/test_taiga_handler.py index bbea108..1a06293 100644 --- a/tests/test_taiga_handler.py +++ b/tests/test_taiga_handler.py @@ -49,7 +49,7 @@ class TestParseTaigaIssueEvent: def test_basic_issue_parsing(self, taiga_issue_payload): from datasources.taiga_handler import parse_taiga_issue_event - result = parse_taiga_issue_event(taiga_issue_payload, "TestPrj") + result = parse_taiga_issue_event(taiga_issue_payload) assert result["event_type"] == "issue" assert result["action_type"] == "create" @@ -87,7 +87,7 @@ def test_issue_assigned_to_none(self): }, "is_closed": False, } - result = parse_taiga_issue_event(payload, "P") + result = parse_taiga_issue_event(payload) assert result["assigned_to"] is None From 36882d4c3422e6247c214bab9089089bc983079f Mon Sep 17 00:00:00 2001 From: EncryptEx <41539618+EncryptEx@users.noreply.github.com> Date: Sat, 14 Mar 2026 14:08:54 +0100 Subject: [PATCH 32/32] refactor: improve logging format in Taiga and GitHub webhook handlers --- datasources/requests/taiga_api_call.py | 2 +- routes/github_routes.py | 26 +++++++++++----- routes/taiga_routes.py | 43 +++++++++++++++++--------- 3 files changed, 47 insertions(+), 24 deletions(-) diff --git a/datasources/requests/taiga_api_call.py b/datasources/requests/taiga_api_call.py index 3e65850..4d3c9f9 100644 --- a/datasources/requests/taiga_api_call.py +++ b/datasources/requests/taiga_api_call.py @@ -39,7 +39,7 @@ def milestone_stats(project_id: str, milestone_id: str, prj: str): if user and psw: token = get_taiga_token(user, psw) headers = {"Authorization": f"Bearer {token}"} - logger.debug("Using Taiga credentials for project:", prj) + logger.debug("Using Taiga credentials for project %s", prj) else: headers = {} logger.info("Using Taiga tunnel without authentication for project: %s", prj) diff --git a/routes/github_routes.py b/routes/github_routes.py index 293702a..9cae38a 100644 --- a/routes/github_routes.py +++ b/routes/github_routes.py @@ -47,7 +47,7 @@ def github_webhook(): # Parse the raw JSON payload using the parse_github_event function parsed_data = parse_github_event(raw_payload, prj) - logger.info(f"Github webhook request processed successfully for team {prj}.") + logger.info("Github webhook request processed successfully for team %s.", prj) if parsed_data.get("ignored"): return make_response(jsonify({"status": "ignored", "event": parsed_data["event"]}), 200) @@ -57,7 +57,12 @@ def github_webhook(): team_name = parsed_data[ "team_name" ] # We wont use this, we will use the external_id instead as its a CENTRALIZED ID - logger.info(f"Processing Github event: {parsed_data['event']} for team: {team_name} (external_id: {prj})") + logger.info( + "Processing Github event: %s for team: %s (external_id: %s)", + parsed_data["event"], + team_name, + prj, + ) event_label = parsed_data["event"] # This is either "commit" or "issue" author_login = parsed_data["sender_info"][ "login" @@ -75,12 +80,15 @@ def github_webhook(): # # COMMUNICATION WITH LD_EVAL USING API logger.info( - f"Notifying LD_EVAL about event: {event_name} for team with external_id: {prj} with quality_model: {quality_model}" + "Notifying LD_EVAL about event: %s for team with external_id: %s with quality_model: %s", + event_name, + prj, + quality_model, ) try: notify_eval_push(event_name, prj, author_login, quality_model) except Exception as e: - logger.error(f"Error notifying LD_EVAL: {e}") + logger.error("Error notifying LD_EVAL: %s", e) return {"status": "error", "message": str(e)}, 500 # If it's a commit push, we may have multiple commits. We need to insert each one separately. @@ -93,9 +101,9 @@ def github_webhook(): commit_doc["event"] = parsed_data["event"] commit_doc["repo_name"] = parsed_data["repo_name"] - logger.debug(f"Inserting commit document: {commit_doc}") + logger.debug("Inserting GitHub commit document for team %s", prj) coll.insert_one(commit_doc) - logger.info(f"Inserting in MongoDB Github commit for team {prj}") + logger.info("Inserting in MongoDB Github commit for team %s", prj) return make_response(jsonify({"status": "ok", "message": "Commits inserted"}), 200) @@ -103,13 +111,15 @@ def github_webhook(): elif "issue" in parsed_data: parsed_data["prj"] = prj coll.insert_one(parsed_data) - logger.info(f"Inserting in MongoDB Github issue for team {prj}") + logger.info("Inserting in MongoDB Github issue for team %s", prj) return make_response(jsonify({"status": "ok", "message": "Issue inserted"}), 200) elif "pull_request" in parsed_data: parsed_data["prj"] = prj coll.insert_one(parsed_data) - logger.info(f"Inserting in MongoDB Github closed pull request for team {prj}") + logger.info( + "Inserting in MongoDB Github closed pull request for team %s", prj + ) return {"status": "ok", "message": "Pull request inserted"}, 200 # If its neither a commit or a issue diff --git a/routes/taiga_routes.py b/routes/taiga_routes.py index f748714..6e78495 100644 --- a/routes/taiga_routes.py +++ b/routes/taiga_routes.py @@ -38,8 +38,12 @@ def taiga_webhook(): event_type = raw_payload.get("type", "") action_type = raw_payload.get("action", "") id = raw_payload.get("data", {}).get("id", "") - team_name = raw_payload.get("data", {}).get("project", {}).get("name", "") - logger.info(f"Processing Taiga event: {event_type} with action: {action_type} for team: {team_name} (external_id: {prj})") + logger.info( + "Processing Taiga event type=%s action=%s project=%s", + event_type, + action_type, + prj, + ) # Decide the Mongo collection name to write to, depending on the event type if event_type in ["userstory", "relateduserstory"]: @@ -57,11 +61,11 @@ def taiga_webhook(): # Handle the deletion of a document before we parse the payload, to avoid data errors if action_type == "delete": - logger.info(f"Deleting document from {collection_name}. ID={id}") + logger.info("Deleting document from %s. ID=%s", collection_name, id) if not id: return jsonify({"error": "No object ID"}), 400 coll.delete_one({f"{event_type}_id": id}) - logger.info(f"Document with {event_type}={id} has been deleted.") + logger.info("Document with %s=%s has been deleted.", event_type, id) return jsonify({"status": "ok"}), 200 # Parse the raw JSON payload using the parse_taiga_event function @@ -79,12 +83,12 @@ def taiga_webhook(): if not user_story_id: return jsonify({"error": "No user story ID"}), 400 - logger.info(f"Upserting user story with ID: {user_story_id}") + logger.info("Upserting user story with ID: %s", user_story_id) parsed_data["prj"] = prj result = coll.update_one( {"userstory_id": user_story_id}, {"$set": parsed_data}, upsert=True ) - logger.info(f"Inserting in MongoDB Taiga userstory for team {prj}") + logger.info("Inserting in MongoDB Taiga userstory for team %s", prj) # If the event is a taks , identify the task ID and upsert/insert it in the collection elif event_type == "task": @@ -97,14 +101,20 @@ def taiga_webhook(): if not task_id: return jsonify({"error": "No task ID"}), 400 - logger.info(f"Upserting task with ID: {task_id}") + logger.info("Upserting task with ID: %s", task_id) # Upsert instead of insert result = parsed_data["prj"] = prj - logger.info(f"Upserting task with ID: {task_id} for team {prj} in collection {collection_name} result: {result}") + logger.info( + "Upserting task with ID: %s for team %s in collection %s result: %s", + task_id, + prj, + collection_name, + result, + ) coll.update_one( {"task_id": task_id}, {"$set": parsed_data}, upsert=True ) - logger.info(f"Inserting in MongoDB Taiga task for team {prj}") + logger.info("Inserting in MongoDB Taiga task for team %s", prj) # If the event is an epic, identify the epic ID and upsert/insert it in the collection elif event_type == "epic": @@ -113,13 +123,13 @@ def taiga_webhook(): if not epic_id: return jsonify({"error": "No epic ID"}), 400 - logger.info(f"Upserting epic with ID: {epic_id}") + logger.info("Upserting epic with ID: %s", epic_id) # Upsert instead of insert parsed_data["prj"] = prj coll.update_one( {"epic_id": epic_id}, {"$set": parsed_data}, upsert=True ) - logger.info(f"Inserting in MongoDB Taiga epic for team {prj}") + logger.info("Inserting in MongoDB Taiga epic for team %s", prj) # If the event is an issue, identify the issue ID and upsert/insert it in the collection elif event_type == "issue": @@ -128,13 +138,13 @@ def taiga_webhook(): if not issue_id: return jsonify({"error": "No issue ID"}), 400 - logger.info(f"Upserting issue with ID: {issue_id}") + logger.info("Upserting issue with ID: %s", issue_id) parsed_data["prj"] = prj # Upsert instead of insert coll.update_one( {"issue_id": issue_id}, {"$set": parsed_data}, upsert=True ) - logger.info(f"Inserting in MongoDB Taiga issue for team {prj}") + logger.info("Inserting in MongoDB Taiga issue for team %s", prj) else: # If the event is not one of the above, we will insert it as a new document @@ -143,12 +153,15 @@ def taiga_webhook(): # COMMUNICATION WITH LD_EVAL USING API logger.info( - f"Notifying LD_EVAL about event: {event_type} for team with external_id: {prj} with quality_model: {quality_model}" + "Notifying LD_EVAL about event: %s for team with external_id: %s with quality_model: %s", + event_type, + prj, + quality_model, ) try: notify_eval_push(event_type, prj, author_login, quality_model) except Exception as e: - logger.error(f"Error notifying LD_EVAL: {e}") + logger.error("Error notifying LD_EVAL: %s", e) return jsonify({"error": "Failed to notify LD_EVAL"}), 500 return jsonify({"status": "ok"}), 200