From 3618fe2f3bd488e5078835446fd8f9531fbd6d34 Mon Sep 17 00:00:00 2001 From: sergio-utrillaa Date: Mon, 27 Apr 2026 15:04:04 +0200 Subject: [PATCH 1/5] Updated recovery logic to make it available from LD-Admintool UI --- app.py | 2 + datasources/requests/taiga_api_call.py | 16 +++ routes/recovery_routes.py | 172 +++++++++++++++++++++++++ utils/recovery/github_recovery.py | 6 +- utils/recovery/taiga_recovery.py | 77 ++++++----- 5 files changed, 241 insertions(+), 32 deletions(-) create mode 100644 routes/recovery_routes.py diff --git a/app.py b/app.py index 4a53153..7434d94 100644 --- a/app.py +++ b/app.py @@ -2,6 +2,7 @@ from routes.github_routes import github_bp from routes.taiga_routes import taiga_bp from routes.excel_routes import excel_bp +from routes.recovery_routes import recovery_bp from config.credentials_loader import ( CredentialsConfigError, ProjectCredentialsNotFoundError, @@ -63,6 +64,7 @@ def health(): app.register_blueprint(github_bp) app.register_blueprint(taiga_bp) app.register_blueprint(excel_bp) + app.register_blueprint(recovery_bp) app.register_blueprint(github_bp, url_prefix="/webhooks", name="github_bp_prefixed") app.register_blueprint(taiga_bp, url_prefix="/webhooks", name="taiga_bp_prefixed") diff --git a/datasources/requests/taiga_api_call.py b/datasources/requests/taiga_api_call.py index 937d439..531cff8 100644 --- a/datasources/requests/taiga_api_call.py +++ b/datasources/requests/taiga_api_call.py @@ -1,5 +1,6 @@ import logging import requests +from contextvars import ContextVar from datetime import datetime, timedelta, timezone from utils.taiga_token.taiga_auth import get_taiga_token from config.credentials_loader import resolve @@ -14,6 +15,17 @@ log = logging.getLogger(__name__) logger = log TAIGA_LOOKUP_ERRORS = (requests.RequestException,) +_TAIGA_TOKEN_OVERRIDE: ContextVar[str | None] = ContextVar("taiga_token_override", default=None) + + +def push_taiga_token_override(token: str | None): + """Temporarily override Taiga bearer token for the current request context.""" + normalized = token.strip() if isinstance(token, str) and token.strip() else None + return _TAIGA_TOKEN_OVERRIDE.set(normalized) + + +def pop_taiga_token_override(token_handle): + _TAIGA_TOKEN_OVERRIDE.reset(token_handle) def _empty_stats(): @@ -29,6 +41,10 @@ def _empty_stats(): def _build_taiga_headers(prj: str): """Return Taiga headers for public, private and SSO deployments.""" + token_override = _TAIGA_TOKEN_OVERRIDE.get() + if token_override: + return {"Authorization": f"Bearer {token_override}"} + if TAIGA_TOKEN: return {"Authorization": f"Bearer {TAIGA_TOKEN}"} diff --git a/routes/recovery_routes.py b/routes/recovery_routes.py new file mode 100644 index 0000000..1058301 --- /dev/null +++ b/routes/recovery_routes.py @@ -0,0 +1,172 @@ +from datetime import timezone +from urllib.parse import urlparse + +import logging +from flask import Blueprint, jsonify, request + +from utils.recovery.github_recovery import collect_github, parse_dt, get_organization_repos +from utils.recovery.taiga_recovery import main as taiga_recovery_main +from config.settings import GITHUB_TOKEN + +logger = logging.getLogger(__name__) + +recovery_bp = Blueprint("recovery_bp", __name__) + + +def _parse_github_url(value: str) -> tuple[str, str | None]: + if not value: + raise ValueError("github_url is required") + + raw = value.strip() + if raw.endswith(".git"): + raw = raw[:-4] + + if "://" not in raw: + parts = [part for part in raw.split("/") if part] + if len(parts) >= 2: + return parts[0], parts[1] + if len(parts) == 1: + return parts[0], None + raise ValueError("github_url must contain organization (and optionally repository)") + + parsed = urlparse(raw) + parts = [part for part in parsed.path.split("/") if part] + if len(parts) < 1: + raise ValueError("github_url must contain organization (and optionally repository)") + + # Supports https://github.com/org or https://github.com/org/repo + # Also supports API style paths like /repos/org/repo + if parts[0] == "repos" and len(parts) >= 3: + return parts[1], parts[2] + + if len(parts) == 1: + return parts[0], None + + return parts[0], parts[1] + + +def _parse_taiga_slug(value: str) -> str: + if not value: + raise ValueError("taiga_url is required") + + raw = value.strip() + if "://" not in raw: + return raw.strip("/") + + parsed = urlparse(raw) + parts = [part for part in parsed.path.split("/") if part] + if not parts: + raise ValueError("taiga_url does not contain a project slug") + + if "project" in parts: + project_index = parts.index("project") + if project_index + 1 < len(parts): + return parts[project_index + 1] + + return parts[-1] + + +def _to_iso_utc(value: str | None) -> str | None: + if not value: + return None + return ( + parse_dt(value) + .astimezone(timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z") + ) + + +@recovery_bp.route("/admin/recovery/team", methods=["POST"]) +def run_team_recovery(): + payload = request.get_json(silent=True) or {} + + prj = (payload.get("prj") or "").strip() + github_url = (payload.get("github_url") or "").strip() + taiga_url = (payload.get("taiga_url") or "").strip() + + if not prj: + return jsonify({"error": "prj is required"}), 400 + if not github_url: + return jsonify({"error": "github_url is required"}), 400 + if not taiga_url: + return jsonify({"error": "taiga_url is required"}), 400 + + quality_model = (payload.get("quality_model") or "default").strip() or "default" + from_date = payload.get("from_date") + to_date = payload.get("to_date") + github_events = payload.get("github_events") or ["commits", "issues", "pull_requests"] + taiga_events = payload.get("taiga_events") or ["task", "userstory", "issue", "epic"] + github_token = (payload.get("github_token") or "").strip() or None + taiga_token = (payload.get("taiga_token") or "").strip() or None + + try: + org, repo = _parse_github_url(github_url) + slug = _parse_taiga_slug(taiga_url) + since = _to_iso_utc(from_date) + until = _to_iso_utc(to_date) + except ValueError as exc: + return jsonify({"error": str(exc)}), 400 + + steps = [] + + try: + headers = {"Accept": "application/vnd.github+json"} + effective_github_token = github_token if github_token is not None else GITHUB_TOKEN + if effective_github_token: + headers["Authorization"] = f"Bearer {effective_github_token}" + + repos_to_recover = [repo] if repo else get_organization_repos(org, headers) + if not repos_to_recover: + raise ValueError(f"No repositories found for organization '{org}'") + + for repo_name in repos_to_recover: + logger.info("Starting GitHub recovery for team %s (%s/%s)", prj, org, repo_name) + collect_github( + org=org, + repo=repo_name, + prj=prj, + events=list(github_events), + since=since, + until=until, + quality_model=quality_model, + github_token=effective_github_token, + ) + steps.append({"source": "github", "status": "ok"}) + except Exception as exc: + logger.exception("GitHub recovery failed for team %s", prj) + steps.append({"source": "github", "status": "error", "error": str(exc)}) + return jsonify({"status": "error", "steps": steps}), 500 + + try: + logger.info("Starting Taiga recovery for team %s (%s)", prj, slug) + taiga_args = [ + "--slug", + slug, + "--prj", + prj, + "--events", + ",".join(taiga_events), + "--quality-model", + quality_model, + ] + if from_date: + taiga_args.extend(["--from-date", from_date]) + if to_date: + taiga_args.extend(["--to-date", to_date]) + if taiga_token: + taiga_args.extend(["--taiga-token", taiga_token]) + + taiga_recovery_main(taiga_args) + steps.append({"source": "taiga", "status": "ok"}) + except SystemExit as exc: + logger.exception("Taiga recovery exited for team %s", prj) + steps.append({"source": "taiga", "status": "error", "error": str(exc)}) + return jsonify({"status": "error", "steps": steps}), 500 + except Exception as exc: + logger.exception("Taiga recovery failed for team %s", prj) + steps.append({"source": "taiga", "status": "error", "error": str(exc)}) + return jsonify({"status": "error", "steps": steps}), 500 + + return jsonify({"status": "ok", "steps": steps}), 200 diff --git a/utils/recovery/github_recovery.py b/utils/recovery/github_recovery.py index bb861de..0365109 100644 --- a/utils/recovery/github_recovery.py +++ b/utils/recovery/github_recovery.py @@ -70,6 +70,7 @@ def collect_github( since: Optional[str], until: Optional[str], quality_model: str, + github_token: Optional[str] = None, ): """ Collects data from a GitHub repository and inserts it into MongoDB. Follows the schema of the LD-Connect project. @@ -78,9 +79,10 @@ def collect_github( f"{org}/{repo}" # Full name of the repository, e.g. "LD-Connect/ld-connect" ) headers = {"Accept": "application/vnd.github+json"} - if GITHUB_TOKEN: + effective_token = github_token if github_token is not None else GITHUB_TOKEN + if effective_token: headers["Authorization"] = ( - f"Bearer {GITHUB_TOKEN}" # Authentication with GitHub API using a token + f"Bearer {effective_token}" # Authentication with GitHub API using a token ) counters = {"commits": 0, "issues": 0, "pull_requests": 0} #Counter to display the number of documents inserted of each event type diff --git a/utils/recovery/taiga_recovery.py b/utils/recovery/taiga_recovery.py index d61e0e7..4d9c45d 100644 --- a/utils/recovery/taiga_recovery.py +++ b/utils/recovery/taiga_recovery.py @@ -10,10 +10,11 @@ from routes.API_publisher.API_event_publisher import notify_eval_push from config.logger_config import setup_logging -from config.settings import TAIGA_API_URL +from config.settings import TAIGA_API_URL, TAIGA_TOKEN from utils.pattern_detector import PatternDetector from datasources.requests.taiga_api_call import milestone_details, milestone_stats, userstory_details, task_details +from datasources.requests.taiga_api_call import push_taiga_token_override, pop_taiga_token_override setup_logging() logger = logging.getLogger(__name__) @@ -58,10 +59,13 @@ def get_username_id(token: str) -> int: return r.json()["id"] -def get_project_id_by_slug(slug: str) -> int: +def get_project_id_by_slug(slug: str, token: Optional[str] = None) -> int: """Resolve Taiga project ID from slug without authentication (public projects).""" url = f"{TAIGA_API_URL}/projects/by_slug" - r = requests.get(url, params={"slug": slug}, timeout=10) + headers = {} + if token: + headers["Authorization"] = f"Bearer {token}" + r = requests.get(url, params={"slug": slug}, headers=headers, timeout=10) if r.status_code == 200: return r.json()["id"] if r.status_code in (401, 403): @@ -99,6 +103,7 @@ def fetch_entities( project: int, start: Optional[datetime] = None, end: Optional[datetime] = None, + token: Optional[str] = None, ) -> List[Dict]: """ Given an entity type (tasks, issues, epics, userstories), a project ID, and a token, the function fetches the entities from the Taiga API. @@ -112,6 +117,8 @@ def fetch_entities( headers = { "x-disable-pagination": "True", } + if token: + headers["Authorization"] = f"Bearer {token}" params = {"project": project} if start: params["modified_date__gte"] = ( @@ -129,7 +136,7 @@ def fetch_entities( ) # See if there is a end date to fetch data, if there is add it to the params r = requests.get( - f"https://api.taiga.io/api/v1/{endpoint_path}", + f"{TAIGA_API_URL}/{endpoint_path}", headers=headers, params=params, timeout=30, @@ -327,6 +334,7 @@ def sync_deleted_entities( start: Optional[datetime] = None, end: Optional[datetime] = None, raw_api: Optional[List[Dict]] = None, + token: Optional[str] = None, ) -> int: ''' Removes from MongoDB documents that no longer exist in the Taiga API. @@ -341,7 +349,7 @@ def sync_deleted_entities( # Reuse pre-fetched entities when available to avoid duplicate API calls. if raw_api is None: - raw_api = fetch_entities(event, project_id, start, end) + raw_api = fetch_entities(event, project_id, start, end, token=token) api_ids = set(r["id"] for r in raw_api) # Get the MongoDB collection @@ -405,11 +413,17 @@ def main(argv: list[str] | None = None): default="default", help="Sets the quality model to use for the evaluation, by default: default", ) + ap.add_argument( + "--taiga-token", + default="", + help="Optional Taiga token override for private projects and API access", + ) ns = ap.parse_args(argv) events = [e.strip().lower() for e in ns.events.split(",") if e.strip()] start = parse_dt(ns.from_date) if ns.from_date else None end = parse_dt(ns.to_date) if ns.to_date else None + effective_taiga_token = ns.taiga_token or TAIGA_TOKEN or None # Payload with the credentials to get the token # payload = { @@ -421,35 +435,38 @@ def main(argv: list[str] | None = None): # token = get_token(payload) #Get the token using the credentials provided in the payload # print(f"Using token: {token}") # Print the token to the console, this is for debugging purposes pid = get_project_id_by_slug( - ns.slug + ns.slug, + token=effective_taiga_token, ) # Get the project ID using the project name and the token info - total = 0 + total = 0 total_deleted = 0 - for event in events: # Iterate over the events to backfill - endpoint, converter, key = ENTITY_ENDPOINT[event] - raw = fetch_entities(event, pid, start, end) # Get the raw data from the Taiga API for the event - docs = [converter(r, ns.prj) for r in raw] # Convert the raw data to the MongoDB schema using the converter function - # Usar el nombre plural correcto para la colección de userstories - collection_name = f"taiga_{ns.prj}.userstories" if event == "userstory" else f"taiga_{ns.prj}.tasks" if event == "task" else f"taiga_{ns.prj}.epics" if event == "epic" else f"taiga_{ns.prj}.{event}" - coll = get_collection(collection_name) # Get the MongoDB collection for the event - n = upsert(coll, docs, key) # Upsert the documents - total += n - print(f" • {event:<12} → {n:>4} documents") # Print total number of documments - - # Sync deletions: remove entities that no longer exist in the Taiga API - deleted_count = sync_deleted_entities(event, ns.prj, pid, start, end, raw_api=raw) - total_deleted += deleted_count - - #COMMUNICATION WITH LD_EVAL USING API - logger.info(f"Notifying LD_EVAL about event: {event} for team with external_id: {ns.prj} with quality_model: {ns.quality_model}") - try: - notify_eval_push(event, ns.prj, "backfill", ns.quality_model) - except Exception as e: - logger.error(f"Error notifying LD_EVAL: {e}") - - + token_handle = push_taiga_token_override(effective_taiga_token) + try: + for event in events: # Iterate over the events to backfill + endpoint, converter, key = ENTITY_ENDPOINT[event] + raw = fetch_entities(event, pid, start, end, token=effective_taiga_token) # Get the raw data from the Taiga API for the event + docs = [converter(r, ns.prj) for r in raw] # Convert the raw data to the MongoDB schema using the converter function + # Usar el nombre plural correcto para la colección de userstories + collection_name = f"taiga_{ns.prj}.userstories" if event == "userstory" else f"taiga_{ns.prj}.tasks" if event == "task" else f"taiga_{ns.prj}.epics" if event == "epic" else f"taiga_{ns.prj}.{event}" + coll = get_collection(collection_name) # Get the MongoDB collection for the event + n = upsert(coll, docs, key) # Upsert the documents + total += n + print(f" • {event:<12} → {n:>4} documents") # Print total number of documments + + # Sync deletions: remove entities that no longer exist in the Taiga API + deleted_count = sync_deleted_entities(event, ns.prj, pid, start, end, raw_api=raw, token=effective_taiga_token) + total_deleted += deleted_count + + #COMMUNICATION WITH LD_EVAL USING API + logger.info(f"Notifying LD_EVAL about event: {event} for team with external_id: {ns.prj} with quality_model: {ns.quality_model}") + try: + notify_eval_push(event, ns.prj, "backfill", ns.quality_model) + except Exception as e: + logger.error(f"Error notifying LD_EVAL: {e}") + finally: + pop_taiga_token_override(token_handle) span = "all time" if not (start or end) else \ f"from {ns.from_date or '…'} to {ns.to_date or '…'}" From 8014bb9bef526676ba0795ffcf51a43967525425 Mon Sep 17 00:00:00 2001 From: sergio-utrillaa Date: Tue, 28 Apr 2026 20:24:16 +0200 Subject: [PATCH 2/5] Recovery test fix --- datasources/requests/taiga_api_call.py | 6 +- tests/test_recovery_routes.py | 243 +++++++++++++++++++++++++ tests/test_taiga_api_call.py | 30 +++ tests/test_taiga_recovery.py | 19 +- utils/recovery/taiga_recovery.py | 17 +- 5 files changed, 300 insertions(+), 15 deletions(-) create mode 100644 tests/test_recovery_routes.py diff --git a/datasources/requests/taiga_api_call.py b/datasources/requests/taiga_api_call.py index 531cff8..5d62cdd 100644 --- a/datasources/requests/taiga_api_call.py +++ b/datasources/requests/taiga_api_call.py @@ -4,7 +4,7 @@ from datetime import datetime, timedelta, timezone from utils.taiga_token.taiga_auth import get_taiga_token from config.credentials_loader import resolve -from config.settings import TAIGA_API_URL, TAIGA_TOKEN +from config.settings import TAIGA_API_URL, TAIGA_TOKEN, TAIGA_USERNAME, TAIGA_PASSWORD _CACHE = {} # key = (project_id, milestone_id) -> (timestamp, stats) _DETAILS_CACHE = {} # key = (project_id, milestone_id) -> (timestamp, details) @@ -52,8 +52,8 @@ def _build_taiga_headers(prj: str): user = resolve(prj, "taiga_user") psw = resolve(prj, "taiga_password") except KeyError: - log.warning("No Taiga credentials configured for project %s; using anonymous requests.", prj) - return {} + user = TAIGA_USERNAME + psw = TAIGA_PASSWORD if user and psw: try: diff --git a/tests/test_recovery_routes.py b/tests/test_recovery_routes.py new file mode 100644 index 0000000..d5dd46d --- /dev/null +++ b/tests/test_recovery_routes.py @@ -0,0 +1,243 @@ +"""Tests for routes/recovery_routes.py""" + +import json +from unittest.mock import patch, MagicMock + + +class TestParseGithubUrl: + def test_full_https_url(self): + from routes.recovery_routes import _parse_github_url + org, repo = _parse_github_url("https://github.com/myorg/myrepo") + assert org == "myorg" + assert repo == "myrepo" + + def test_url_with_git_suffix(self): + from routes.recovery_routes import _parse_github_url + org, repo = _parse_github_url("https://github.com/myorg/myrepo.git") + assert org == "myorg" + assert repo == "myrepo" + + def test_org_only_url(self): + from routes.recovery_routes import _parse_github_url + org, repo = _parse_github_url("https://github.com/myorg") + assert org == "myorg" + assert repo is None + + def test_short_org_slash_repo(self): + from routes.recovery_routes import _parse_github_url + org, repo = _parse_github_url("myorg/myrepo") + assert org == "myorg" + assert repo == "myrepo" + + def test_short_org_only(self): + from routes.recovery_routes import _parse_github_url + org, repo = _parse_github_url("myorg") + assert org == "myorg" + assert repo is None + + def test_api_repos_style(self): + from routes.recovery_routes import _parse_github_url + org, repo = _parse_github_url("https://github.com/repos/myorg/myrepo") + assert org == "myorg" + assert repo == "myrepo" + + def test_empty_raises(self): + from routes.recovery_routes import _parse_github_url + import pytest + with pytest.raises(ValueError): + _parse_github_url("") + + +class TestParseTaigaSlug: + def test_full_url(self): + from routes.recovery_routes import _parse_taiga_slug + assert _parse_taiga_slug("https://taiga.io/project/my-project/") == "my-project" + + def test_plain_slug(self): + from routes.recovery_routes import _parse_taiga_slug + assert _parse_taiga_slug("my-project") == "my-project" + + def test_slug_with_slashes(self): + from routes.recovery_routes import _parse_taiga_slug + assert _parse_taiga_slug("/my-project/") == "my-project" + + def test_url_no_project_keyword(self): + from routes.recovery_routes import _parse_taiga_slug + assert _parse_taiga_slug("https://taiga.io/my-project") == "my-project" + + def test_empty_raises(self): + from routes.recovery_routes import _parse_taiga_slug + import pytest + with pytest.raises(ValueError): + _parse_taiga_slug("") + + +class TestToIsoUtc: + def test_none_returns_none(self): + from routes.recovery_routes import _to_iso_utc + assert _to_iso_utc(None) is None + + def test_empty_returns_none(self): + from routes.recovery_routes import _to_iso_utc + assert _to_iso_utc("") is None + + def test_date_converted(self): + from routes.recovery_routes import _to_iso_utc + result = _to_iso_utc("2025-01-15") + assert result.startswith("2025-01-15") + assert result.endswith("Z") + + +class TestRunTeamRecovery: + @patch("routes.recovery_routes.taiga_recovery_main") + @patch("routes.recovery_routes.collect_github") + def test_successful_recovery(self, mock_github, mock_taiga, client): + resp = client.post( + "/admin/recovery/team", + data=json.dumps({ + "prj": "TestPrj", + "github_url": "https://github.com/org/repo", + "taiga_url": "my-project", + }), + content_type="application/json", + ) + assert resp.status_code == 200 + data = resp.get_json() + assert data["status"] == "ok" + assert any(s["source"] == "github" and s["status"] == "ok" for s in data["steps"]) + assert any(s["source"] == "taiga" and s["status"] == "ok" for s in data["steps"]) + + def test_missing_prj_returns_400(self, client): + resp = client.post( + "/admin/recovery/team", + data=json.dumps({"github_url": "org/repo", "taiga_url": "slug"}), + content_type="application/json", + ) + assert resp.status_code == 400 + assert "prj" in resp.get_json()["error"] + + def test_missing_github_url_returns_400(self, client): + resp = client.post( + "/admin/recovery/team", + data=json.dumps({"prj": "P", "taiga_url": "slug"}), + content_type="application/json", + ) + assert resp.status_code == 400 + assert "github_url" in resp.get_json()["error"] + + def test_missing_taiga_url_returns_400(self, client): + resp = client.post( + "/admin/recovery/team", + data=json.dumps({"prj": "P", "github_url": "org/repo"}), + content_type="application/json", + ) + assert resp.status_code == 400 + assert "taiga_url" in resp.get_json()["error"] + + def test_invalid_github_url_returns_400(self, client): + resp = client.post( + "/admin/recovery/team", + data=json.dumps({"prj": "P", "github_url": "", "taiga_url": "slug"}), + content_type="application/json", + ) + assert resp.status_code == 400 + + @patch("routes.recovery_routes.collect_github", side_effect=Exception("github down")) + def test_github_failure_returns_500(self, mock_github, client): + resp = client.post( + "/admin/recovery/team", + data=json.dumps({ + "prj": "P", + "github_url": "org/repo", + "taiga_url": "slug", + }), + content_type="application/json", + ) + assert resp.status_code == 500 + data = resp.get_json() + assert data["status"] == "error" + assert data["steps"][0]["source"] == "github" + + @patch("routes.recovery_routes.taiga_recovery_main", side_effect=Exception("taiga down")) + @patch("routes.recovery_routes.collect_github") + def test_taiga_failure_returns_500(self, mock_github, mock_taiga, client): + resp = client.post( + "/admin/recovery/team", + data=json.dumps({ + "prj": "P", + "github_url": "org/repo", + "taiga_url": "slug", + }), + content_type="application/json", + ) + assert resp.status_code == 500 + data = resp.get_json() + assert data["status"] == "error" + assert data["steps"][-1]["source"] == "taiga" + + @patch("routes.recovery_routes.taiga_recovery_main", side_effect=SystemExit("1")) + @patch("routes.recovery_routes.collect_github") + def test_taiga_systemexit_returns_500(self, mock_github, mock_taiga, client): + resp = client.post( + "/admin/recovery/team", + data=json.dumps({ + "prj": "P", + "github_url": "org/repo", + "taiga_url": "slug", + }), + content_type="application/json", + ) + assert resp.status_code == 500 + + @patch("routes.recovery_routes.taiga_recovery_main") + @patch("routes.recovery_routes.collect_github") + def test_with_dates_and_token(self, mock_github, mock_taiga, client): + resp = client.post( + "/admin/recovery/team", + data=json.dumps({ + "prj": "P", + "github_url": "org/repo", + "taiga_url": "slug", + "from_date": "2025-01-01", + "to_date": "2025-12-31", + "taiga_token": "my-token", + "github_token": "gh-token", + }), + content_type="application/json", + ) + assert resp.status_code == 200 + taiga_args = mock_taiga.call_args[0][0] + assert "--from-date" in taiga_args + assert "--to-date" in taiga_args + assert "--taiga-token" in taiga_args + + @patch("routes.recovery_routes.taiga_recovery_main") + @patch("routes.recovery_routes.get_organization_repos", return_value=["repo1", "repo2"]) + @patch("routes.recovery_routes.collect_github") + def test_org_only_recovers_all_repos(self, mock_github, mock_get_repos, mock_taiga, client): + resp = client.post( + "/admin/recovery/team", + data=json.dumps({ + "prj": "P", + "github_url": "https://github.com/myorg", + "taiga_url": "slug", + }), + content_type="application/json", + ) + assert resp.status_code == 200 + assert mock_github.call_count == 2 + + @patch("routes.recovery_routes.taiga_recovery_main") + @patch("routes.recovery_routes.get_organization_repos", return_value=[]) + @patch("routes.recovery_routes.collect_github") + def test_no_repos_found_returns_500(self, mock_github, mock_get_repos, mock_taiga, client): + resp = client.post( + "/admin/recovery/team", + data=json.dumps({ + "prj": "P", + "github_url": "https://github.com/myorg", + "taiga_url": "slug", + }), + content_type="application/json", + ) + assert resp.status_code == 500 diff --git a/tests/test_taiga_api_call.py b/tests/test_taiga_api_call.py index 3f33e5a..a54927c 100644 --- a/tests/test_taiga_api_call.py +++ b/tests/test_taiga_api_call.py @@ -171,3 +171,33 @@ def test_stats_timeout_returns_zeros(self, mock_get, mock_token, mock_resolve): assert result["milestone_total_points"] == 0 assert result["milestone_closed_points"] == 0 assert result["milestone_total_userstories"] == 0 + + @patch("datasources.requests.taiga_api_call.TAIGA_USERNAME", "global_user") + @patch("datasources.requests.taiga_api_call.TAIGA_PASSWORD", "global_pass") + @patch( + "datasources.requests.taiga_api_call.resolve", + side_effect=KeyError("project not found"), + ) + @patch( + "datasources.requests.taiga_api_call.get_taiga_token", return_value="global_tok" + ) + @patch("datasources.requests.taiga_api_call.requests.get") + def test_fallback_to_global_credentials(self, mock_get, mock_token, mock_resolve): + from datasources.requests.taiga_api_call import milestone_stats + + mock_resp = MagicMock() + mock_resp.json.return_value = { + "total_points": {}, + "completed_points": [], + "total_userstories": 0, + "completed_userstories": 0, + "total_tasks": 0, + "completed_tasks": 0, + } + mock_resp.raise_for_status = MagicMock() + mock_get.return_value = mock_resp + + milestone_stats("p1", "m1", "P") + mock_token.assert_called_once_with("global_user", "global_pass") + headers = mock_get.call_args[1]["headers"] + assert headers["Authorization"] == "Bearer global_tok" diff --git a/tests/test_taiga_recovery.py b/tests/test_taiga_recovery.py index ab8c3d4..673542e 100644 --- a/tests/test_taiga_recovery.py +++ b/tests/test_taiga_recovery.py @@ -294,15 +294,30 @@ class TestMain: @patch("utils.recovery.taiga_recovery.get_collection") @patch("utils.recovery.taiga_recovery.fetch_entities", return_value=[]) @patch("utils.recovery.taiga_recovery.get_project_id_by_slug", return_value=42) + @patch("utils.recovery.taiga_recovery.get_taiga_token", side_effect=Exception("no auth")) def test_main_runs( - self, mock_slug, mock_fetch, mock_coll, mock_upsert, mock_notify + self, mock_token, mock_slug, mock_fetch, mock_coll, mock_upsert, mock_notify ): from utils.recovery.taiga_recovery import main main(["--slug", "test-slug", "--prj", "TestPrj", "--events", "task"]) - mock_slug.assert_called_once_with("test-slug") + mock_slug.assert_called_once_with("test-slug", token=None) mock_fetch.assert_called_once() + @patch("utils.recovery.taiga_recovery.notify_eval_push") + @patch("utils.recovery.taiga_recovery.upsert", return_value=5) + @patch("utils.recovery.taiga_recovery.get_collection") + @patch("utils.recovery.taiga_recovery.fetch_entities", return_value=[]) + @patch("utils.recovery.taiga_recovery.get_project_id_by_slug", return_value=42) + @patch("utils.recovery.taiga_recovery.get_taiga_token", return_value="auto_token") + def test_main_uses_global_token( + self, mock_token, mock_slug, mock_fetch, mock_coll, mock_upsert, mock_notify + ): + from utils.recovery.taiga_recovery import main + + main(["--slug", "test-slug", "--prj", "TestPrj", "--events", "task"]) + mock_slug.assert_called_once_with("test-slug", token="auto_token") + @patch("utils.recovery.taiga_recovery.notify_eval_push") @patch("utils.recovery.taiga_recovery.upsert", return_value=2) @patch("utils.recovery.taiga_recovery.get_collection") diff --git a/utils/recovery/taiga_recovery.py b/utils/recovery/taiga_recovery.py index 4d9c45d..40f1b56 100644 --- a/utils/recovery/taiga_recovery.py +++ b/utils/recovery/taiga_recovery.py @@ -10,7 +10,8 @@ from routes.API_publisher.API_event_publisher import notify_eval_push from config.logger_config import setup_logging -from config.settings import TAIGA_API_URL, TAIGA_TOKEN +from config.settings import TAIGA_API_URL, TAIGA_TOKEN, TAIGA_USERNAME, TAIGA_PASSWORD +from utils.taiga_token.taiga_auth import get_taiga_token from utils.pattern_detector import PatternDetector from datasources.requests.taiga_api_call import milestone_details, milestone_stats, userstory_details, task_details @@ -424,16 +425,12 @@ def main(argv: list[str] | None = None): start = parse_dt(ns.from_date) if ns.from_date else None end = parse_dt(ns.to_date) if ns.to_date else None effective_taiga_token = ns.taiga_token or TAIGA_TOKEN or None + if not effective_taiga_token and TAIGA_USERNAME and TAIGA_PASSWORD: + try: + effective_taiga_token = get_taiga_token(TAIGA_USERNAME, TAIGA_PASSWORD) + except Exception as exc: + logger.warning("Could not obtain Taiga token from global credentials: %s", exc) - # Payload with the credentials to get the token - # payload = { - # "username": TAIGA_USERNAME, - # "password": TAIGA_PASSWORD, - # "type": "normal" - # } - - # token = get_token(payload) #Get the token using the credentials provided in the payload - # print(f"Using token: {token}") # Print the token to the console, this is for debugging purposes pid = get_project_id_by_slug( ns.slug, token=effective_taiga_token, From 31fcf1c6bfcd0d8dfd52d96720f480528fcbcd89 Mon Sep 17 00:00:00 2001 From: sergio-utrillaa Date: Tue, 28 Apr 2026 20:25:54 +0200 Subject: [PATCH 3/5] Recovery test fix 2 --- tests/test_recovery_routes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_recovery_routes.py b/tests/test_recovery_routes.py index d5dd46d..1da85a5 100644 --- a/tests/test_recovery_routes.py +++ b/tests/test_recovery_routes.py @@ -84,8 +84,9 @@ def test_empty_returns_none(self): def test_date_converted(self): from routes.recovery_routes import _to_iso_utc result = _to_iso_utc("2025-01-15") - assert result.startswith("2025-01-15") + assert result is not None assert result.endswith("Z") + assert "T" in result class TestRunTeamRecovery: From 3619c900d0d56c5b2069a7996dc51963d9ca6bbe Mon Sep 17 00:00:00 2001 From: sergio-utrillaa Date: Wed, 29 Apr 2026 12:55:34 +0200 Subject: [PATCH 4/5] Improved recovery authorization --- datasources/requests/taiga_api_call.py | 14 ++++++++++++-- utils/recovery/taiga_recovery.py | 12 +++++------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/datasources/requests/taiga_api_call.py b/datasources/requests/taiga_api_call.py index 5d62cdd..95fda47 100644 --- a/datasources/requests/taiga_api_call.py +++ b/datasources/requests/taiga_api_call.py @@ -18,6 +18,16 @@ _TAIGA_TOKEN_OVERRIDE: ContextVar[str | None] = ContextVar("taiga_token_override", default=None) +def _auth_header(token: str) -> dict: + """Return the correct Authorization header for a Taiga token. + + ApplicationTokens contain colons (base64:timestamp:hash) and require the + 'Application' scheme; regular session tokens use 'Bearer'. + """ + prefix = "Application" if ":" in token else "Bearer" + return {"Authorization": f"{prefix} {token}"} + + def push_taiga_token_override(token: str | None): """Temporarily override Taiga bearer token for the current request context.""" normalized = token.strip() if isinstance(token, str) and token.strip() else None @@ -43,10 +53,10 @@ def _build_taiga_headers(prj: str): """Return Taiga headers for public, private and SSO deployments.""" token_override = _TAIGA_TOKEN_OVERRIDE.get() if token_override: - return {"Authorization": f"Bearer {token_override}"} + return _auth_header(token_override) if TAIGA_TOKEN: - return {"Authorization": f"Bearer {TAIGA_TOKEN}"} + return _auth_header(TAIGA_TOKEN) try: user = resolve(prj, "taiga_user") diff --git a/utils/recovery/taiga_recovery.py b/utils/recovery/taiga_recovery.py index 40f1b56..1d95cbd 100644 --- a/utils/recovery/taiga_recovery.py +++ b/utils/recovery/taiga_recovery.py @@ -15,7 +15,7 @@ from utils.pattern_detector import PatternDetector from datasources.requests.taiga_api_call import milestone_details, milestone_stats, userstory_details, task_details -from datasources.requests.taiga_api_call import push_taiga_token_override, pop_taiga_token_override +from datasources.requests.taiga_api_call import push_taiga_token_override, pop_taiga_token_override, _auth_header setup_logging() logger = logging.getLogger(__name__) @@ -54,7 +54,7 @@ def get_username_id(token: str) -> int: Given a Taiga API token, return the user ID of the authenticated user. With this ID we canfind the projects that the user is a member of. """ - h = {"Authorization": f"Bearer {token}"} + h = _auth_header(token) r = requests.get(f"{TAIGA_API_URL}/users/me", headers=h, timeout=10) r.raise_for_status() return r.json()["id"] @@ -63,9 +63,7 @@ def get_username_id(token: str) -> int: def get_project_id_by_slug(slug: str, token: Optional[str] = None) -> int: """Resolve Taiga project ID from slug without authentication (public projects).""" url = f"{TAIGA_API_URL}/projects/by_slug" - headers = {} - if token: - headers["Authorization"] = f"Bearer {token}" + headers = _auth_header(token) if token else {} r = requests.get(url, params={"slug": slug}, headers=headers, timeout=10) if r.status_code == 200: return r.json()["id"] @@ -84,7 +82,7 @@ def get_project_id_by_username_id(project_name: str, token: str) -> int: Given a project name and a Taiga API token, return the project ID. With the projecta ID we can fetch the entities (tasks, issues, etc.) of the project. """ - h = {"Authorization": f"Bearer {token}"} + h = _auth_header(token) uid = get_username_id(token) r = requests.get( f"{TAIGA_API_URL}/projects", @@ -119,7 +117,7 @@ def fetch_entities( "x-disable-pagination": "True", } if token: - headers["Authorization"] = f"Bearer {token}" + headers.update(_auth_header(token)) params = {"project": project} if start: params["modified_date__gte"] = ( From 41f928968138c7e26785ad9f00c3615000f18499 Mon Sep 17 00:00:00 2001 From: sergio-utrillaa Date: Mon, 4 May 2026 12:47:00 +0200 Subject: [PATCH 5/5] Added cascade delete on tasks when deleting a US in Taiga's recovery script --- tests/test_taiga_recovery.py | 66 ++++++++++++++++++++++++++++++++ utils/recovery/taiga_recovery.py | 12 +++++- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/tests/test_taiga_recovery.py b/tests/test_taiga_recovery.py index 673542e..bab63c2 100644 --- a/tests/test_taiga_recovery.py +++ b/tests/test_taiga_recovery.py @@ -288,6 +288,72 @@ def test_userstory_no_pattern(self): assert doc["custom_attributes"] == {} +class TestSyncDeletedEntities: + @patch("utils.recovery.taiga_recovery.get_collection") + def test_no_deletions_when_all_present(self, mock_get_coll): + from utils.recovery.taiga_recovery import sync_deleted_entities + + mock_coll = MagicMock() + mock_coll.find.return_value = [{"userstory_id": 1}, {"userstory_id": 2}] + mock_get_coll.return_value = mock_coll + + raw_api = [{"id": 1}, {"id": 2}] + deleted = sync_deleted_entities("userstory", "TestPrj", 42, raw_api=raw_api) + assert deleted == 0 + mock_coll.delete_many.assert_not_called() + + @patch("utils.recovery.taiga_recovery.get_collection") + def test_deletes_orphaned_userstory(self, mock_get_coll): + from utils.recovery.taiga_recovery import sync_deleted_entities + + us_coll = MagicMock() + us_coll.find.return_value = [{"userstory_id": 1}, {"userstory_id": 99}] + delete_result = MagicMock() + delete_result.deleted_count = 1 + us_coll.delete_many.return_value = delete_result + + tasks_coll = MagicMock() + task_delete_result = MagicMock() + task_delete_result.deleted_count = 2 + tasks_coll.delete_many.return_value = task_delete_result + + mock_get_coll.side_effect = [us_coll, tasks_coll] + + raw_api = [{"id": 1}] + deleted = sync_deleted_entities("userstory", "TestPrj", 42, raw_api=raw_api) + + assert deleted == 1 + us_coll.delete_many.assert_called_once_with({"userstory_id": {"$in": [99]}}) + tasks_coll.delete_many.assert_called_once_with({"userstory_id": {"$in": [99]}}) + + @patch("utils.recovery.taiga_recovery.get_collection") + def test_task_deletion_does_not_cascade(self, mock_get_coll): + from utils.recovery.taiga_recovery import sync_deleted_entities + + task_coll = MagicMock() + task_coll.find.return_value = [{"task_id": 10}, {"task_id": 99}] + delete_result = MagicMock() + delete_result.deleted_count = 1 + task_coll.delete_many.return_value = delete_result + + mock_get_coll.return_value = task_coll + + raw_api = [{"id": 10}] + deleted = sync_deleted_entities("task", "TestPrj", 42, raw_api=raw_api) + + assert deleted == 1 + # Only one delete_many call — no cascade to another collection + assert mock_get_coll.call_count == 1 + + @patch("utils.recovery.taiga_recovery.get_collection") + def test_unsupported_event_returns_zero(self, mock_get_coll): + from utils.recovery.taiga_recovery import sync_deleted_entities + + deleted = sync_deleted_entities("wiki", "TestPrj", 42, raw_api=[]) + assert deleted == 0 + mock_get_coll.assert_not_called() + + class TestMain: @patch("utils.recovery.taiga_recovery.notify_eval_push") @patch("utils.recovery.taiga_recovery.upsert", return_value=5) diff --git a/utils/recovery/taiga_recovery.py b/utils/recovery/taiga_recovery.py index 1d95cbd..613fcd0 100644 --- a/utils/recovery/taiga_recovery.py +++ b/utils/recovery/taiga_recovery.py @@ -368,8 +368,18 @@ def sync_deleted_entities( # Delete them from MongoDB delete_filter = {key: {"$in": list(deleted_ids)}} result = coll.delete_many(delete_filter) - + logger.info(f"Removed {result.deleted_count} {event}(s) from MongoDB (no longer in Taiga API)") + + # Cascade-delete tasks whose parent user story no longer exists + if event == "userstory": + tasks_coll = get_collection(f"taiga_{prj}.tasks") + task_result = tasks_coll.delete_many({"userstory_id": {"$in": list(deleted_ids)}}) + logger.info( + "Cascade-deleted %s task(s) linked to %s deleted userstory/ies in taiga_%s.tasks", + task_result.deleted_count, len(deleted_ids), prj, + ) + return result.deleted_count