diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..403bd7e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,124 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main, dev] + + + +jobs: + test: + permissions: + contents: read + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.14" + cache: pip + cache-dependency-path: requirements.txt + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install pytest pytest-cov + + - name: Run tests with coverage + run: pytest --cov=. --cov-report=term --cov-report=xml --cov-report=html + + - name: Upload coverage artifacts + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: | + coverage.xml + htmlcov/ + if-no-files-found: error + + lint: + permissions: + contents: read + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.14" + cache: pip + + - name: Install Ruff + run: | + python -m pip install --upgrade pip + pip install ruff + + - name: Run Ruff + run: ruff check . --exclude tests,deprecated + + bandit: + permissions: + contents: read + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.14" + cache: pip + + - name: Install Bandit + run: | + python -m pip install --upgrade pip + pip install bandit + + - name: Run Bandit + run: bandit -r . -x tests -ll -x tests,deprecated + + gitleaks: + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - name: Checkout repository (full history) + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install Gitleaks (via cli) + run: | + curl -sSL -o /tmp/gitleaks.tar.gz https://github.com/gitleaks/gitleaks/releases/download/v8.30.0/gitleaks_8.30.0_linux_x64.tar.gz + tar xz -C /tmp -f /tmp/gitleaks.tar.gz + sudo mv /tmp/gitleaks /usr/local/bin/gitleaks + sudo chmod +x /usr/local/bin/gitleaks + + - name: Run Gitleaks + run: gitleaks detect --source . --redact --verbose --baseline-path .gitleaks-baseline.json --exit-code 1 + + semgrep: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Run Semgrep + uses: semgrep/semgrep-action@v1 + with: + config: p/security-audit diff --git a/.github/workflows/main-pr.yml b/.github/workflows/main-pr.yml new file mode 100644 index 0000000..bc61ee5 --- /dev/null +++ b/.github/workflows/main-pr.yml @@ -0,0 +1,20 @@ +name: Restrict PR Source + +on: + pull_request: + branches: + - main + +jobs: + check-branch: + runs-on: ubuntu-latest + env: + GITHUB_HEAD_REF: ${{ github.head_ref }} + steps: + + - name: Fail if not from develop + run: | + if [[ $GITHUB_HEAD_REF != "dev" ]]; then + echo "PR must come from develop branch only." + exit 1 + fi diff --git a/.gitignore b/.gitignore index 695768a..6056d80 100644 --- a/.gitignore +++ b/.gitignore @@ -195,9 +195,9 @@ cython_debug/ .abstra/ # Visual Studio Code -# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore # that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore -# and can be added to the global gitignore or merged into this file. However, if you prefer, +# and can be added to the global gitignore or merged into this file. However, if you prefer, # you could uncomment the following to ignore the entire vscode folder # .vscode/ @@ -217,3 +217,4 @@ __marimo__/ # custom config_files/credentials_config.json +.vscode/mcp.json diff --git a/.gitleaks-baseline.json b/.gitleaks-baseline.json new file mode 100644 index 0000000..afbd011 --- /dev/null +++ b/.gitleaks-baseline.json @@ -0,0 +1,443 @@ +[ + { + "RuleID": "github-pat", + "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "StartLine": 10, + "EndLine": 10, + "StartColumn": 23, + "EndColumn": 62, + "Match": "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", + "Secret": "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", + "File": "config_files/credentials_config.json", + "SymlinkFile": "", + "Commit": "51732615a354fe996d6892942eb0fd631c7570b3", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/51732615a354fe996d6892942eb0fd631c7570b3/config_files/credentials_config.json#L10", + "Entropy": 4.753056, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-07-01T14:36:18Z", + "Message": "Final modifications", + "Tags": [], + "Fingerprint": "51732615a354fe996d6892942eb0fd631c7570b3:config_files/credentials_config.json:github-pat:10" + }, + { + "RuleID": "jwt", + "Description": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", + "StartLine": 36, + "EndLine": 36, + "StartColumn": 11, + "EndColumn": 522, + "Match": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQ3MzMxMDk0LCJqdGkiOiI4N2U1NWM5MTNmMzU0NjQwODU1MTFiMWMwY2I0YzM3NSIsInVzZXJfaWQiOjc1OTg5Nn0.Rh57RIdpOZLQ_xg_5c22dLIp3yXvvWB-aC2RVwgLdvSOirkhIBgqDKBXQ3j3OlbEi4BIgD-WfZZR6CXtyewOnX8ov4RCtTtdpxpuo8lch4ZhqPuvZ-UT-w8ytenrcxZoH3vz3ikUaevYDbjuCV3FSoiWn1Xxcg8jdiu-bsx-nenZ7GhydvE6VCKogF29bPjLUuZbkk-BtxVHiTPDEb6qOWx7wo83b4Io8D0zaKxgVQRzliUUy-my8HdWTex-ELyaIwzWVAkzbYGh7DjmRY4opGqFovmDkOCCOmv8Ycm3VU2RqFd7nfJAEZayMkwe1l481dvmPKMfcJ0llORocbkC0A\"", + "Secret": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQ3MzMxMDk0LCJqdGkiOiI4N2U1NWM5MTNmMzU0NjQwODU1MTFiMWMwY2I0YzM3NSIsInVzZXJfaWQiOjc1OTg5Nn0.Rh57RIdpOZLQ_xg_5c22dLIp3yXvvWB-aC2RVwgLdvSOirkhIBgqDKBXQ3j3OlbEi4BIgD-WfZZR6CXtyewOnX8ov4RCtTtdpxpuo8lch4ZhqPuvZ-UT-w8ytenrcxZoH3vz3ikUaevYDbjuCV3FSoiWn1Xxcg8jdiu-bsx-nenZ7GhydvE6VCKogF29bPjLUuZbkk-BtxVHiTPDEb6qOWx7wo83b4Io8D0zaKxgVQRzliUUy-my8HdWTex-ELyaIwzWVAkzbYGh7DjmRY4opGqFovmDkOCCOmv8Ycm3VU2RqFd7nfJAEZayMkwe1l481dvmPKMfcJ0llORocbkC0A", + "File": "utils/taiga_get_milestone_points.py", + "SymlinkFile": "", + "Commit": "81c19ba5b4da57e7b234a97a75369c537a122880", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/81c19ba5b4da57e7b234a97a75369c537a122880/utils/taiga_get_milestone_points.py#L36", + "Entropy": 5.883454, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-05-19T23:42:51Z", + "Message": "milestone_points", + "Tags": [], + "Fingerprint": "81c19ba5b4da57e7b234a97a75369c537a122880:utils/taiga_get_milestone_points.py:jwt:36" + }, + { + "RuleID": "jwt", + "Description": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", + "StartLine": 37, + "EndLine": 37, + "StartColumn": 11, + "EndColumn": 522, + "Match": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQ3MzA3MDI1LCJqdGkiOiIwMzA5NDcyNzZlMjc0YzQxYWM0N2NiYTllNzE2NGYyMyIsInVzZXJfaWQiOjc1OTg5Nn0.bYh7cfGv6WzhpdQB1nXu75oEI3uklWvEQAkoItY5R9j0WPutzVcXbAbeXkBH-sfJa6k-QjVCKfFYruqGSH4819q7tCYzc67sqgiA0MTsJkKuzUa_2aa1owyHMtiDYGMK3ZhN8W7KQxarMEvHXEtyrUKBI5gU_ewUoqtLBcplCBSFfrIvC9UqjA7OzS3YlBaS9YCYP3vt0ndg_qGRq1hqb64sByx7eld_6z-1Rm2KmfqHztqj6miR4K3KhNlO45lyNti_WE4nZJxfku4yXo8G91MSVFQSCZgLXUvRL-DO0b28Fu6LQX5T9or3cDNvuaGuT0SP3A1Jf-KIYU21sWYnrA\"", + "Secret": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQ3MzA3MDI1LCJqdGkiOiIwMzA5NDcyNzZlMjc0YzQxYWM0N2NiYTllNzE2NGYyMyIsInVzZXJfaWQiOjc1OTg5Nn0.bYh7cfGv6WzhpdQB1nXu75oEI3uklWvEQAkoItY5R9j0WPutzVcXbAbeXkBH-sfJa6k-QjVCKfFYruqGSH4819q7tCYzc67sqgiA0MTsJkKuzUa_2aa1owyHMtiDYGMK3ZhN8W7KQxarMEvHXEtyrUKBI5gU_ewUoqtLBcplCBSFfrIvC9UqjA7OzS3YlBaS9YCYP3vt0ndg_qGRq1hqb64sByx7eld_6z-1Rm2KmfqHztqj6miR4K3KhNlO45lyNti_WE4nZJxfku4yXo8G91MSVFQSCZgLXUvRL-DO0b28Fu6LQX5T9or3cDNvuaGuT0SP3A1Jf-KIYU21sWYnrA", + "File": "test_points.py", + "SymlinkFile": "", + "Commit": "81c19ba5b4da57e7b234a97a75369c537a122880", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/81c19ba5b4da57e7b234a97a75369c537a122880/test_points.py#L37", + "Entropy": 5.9098024, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-05-19T23:42:51Z", + "Message": "milestone_points", + "Tags": [], + "Fingerprint": "81c19ba5b4da57e7b234a97a75369c537a122880:test_points.py:jwt:37" + }, + { + "RuleID": "github-pat", + "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "StartLine": 5, + "EndLine": 5, + "StartColumn": 22, + "EndColumn": 61, + "Match": "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", + "Secret": "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", + "File": "test.py", + "SymlinkFile": "", + "Commit": "4c29834f8242fe60c28182f982df7c9616923cf4", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/4c29834f8242fe60c28182f982df7c9616923cf4/test.py#L5", + "Entropy": 4.803056, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-05-27T00:26:14Z", + "Message": "recovry", + "Tags": [], + "Fingerprint": "4c29834f8242fe60c28182f982df7c9616923cf4:test.py:github-pat:5" + }, + { + "RuleID": "github-pat", + "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "StartLine": 40, + "EndLine": 40, + "StartColumn": 11, + "EndColumn": 50, + "Match": "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", + "Secret": "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", + "File": "test.py", + "SymlinkFile": "", + "Commit": "4c29834f8242fe60c28182f982df7c9616923cf4", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/4c29834f8242fe60c28182f982df7c9616923cf4/test.py#L40", + "Entropy": 4.803056, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-05-27T00:26:14Z", + "Message": "recovry", + "Tags": [], + "Fingerprint": "4c29834f8242fe60c28182f982df7c9616923cf4:test.py:github-pat:40" + }, + { + "RuleID": "github-pat", + "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "StartLine": 12, + "EndLine": 12, + "StartColumn": 18, + "EndColumn": 57, + "Match": "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", + "Secret": "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", + "File": "env", + "SymlinkFile": "", + "Commit": "4bbb59be36182bdab8829924c311a6109bc30748", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/4bbb59be36182bdab8829924c311a6109bc30748/env#L12", + "Entropy": 4.8341837, + "Author": "Pablo", + "Email": "pgomezna@gmail.com", + "Date": "2025-04-04T17:32:22Z", + "Message": "API change", + "Tags": [], + "Fingerprint": "4bbb59be36182bdab8829924c311a6109bc30748:env:github-pat:12" + }, + { + "RuleID": "jwt", + "Description": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", + "StartLine": 17, + "EndLine": 17, + "StartColumn": 17, + "EndColumn": 528, + "Match": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQyOTQxMDg1LCJqdGkiOiJiMzJhZjE1NDU4NzI0OThhOTY2NWJhMWVkOTIxZjMzYSIsInVzZXJfaWQiOjc1OTg5Nn0.H8ug241uZP6q2AbSKSZZbcxRSlRE52wGwok8FfZGhVjnshGqGkbEJnhodESqI94bT6bPCToXxDeLqLEdwQDMz7t5j7vDioxyG8cxF2E2bjLVLt6-wkT6U1mdomyyYvpP78gkmlNPr0a3Jv60MplTgcSnfrQ11N2xYDzf1yNtTJsYCjZKzy5d9zqsOkILBKpqnpVu5FcJlXdYyc-K0TMNsWbi0zsA3F7OBchEfOREZcdwZNfN7Qv4s5pb38ZdPQsfD_wGrvjzgE5FdOOvV7-9gEREI7ruytTawsoHSLMd_lwxIfXBUUJxJSAb13oDKEGWY5N9fVN0B3ayuRtwjZF-nw\"", + "Secret": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQyOTQxMDg1LCJqdGkiOiJiMzJhZjE1NDU4NzI0OThhOTY2NWJhMWVkOTIxZjMzYSIsInVzZXJfaWQiOjc1OTg5Nn0.H8ug241uZP6q2AbSKSZZbcxRSlRE52wGwok8FfZGhVjnshGqGkbEJnhodESqI94bT6bPCToXxDeLqLEdwQDMz7t5j7vDioxyG8cxF2E2bjLVLt6-wkT6U1mdomyyYvpP78gkmlNPr0a3Jv60MplTgcSnfrQ11N2xYDzf1yNtTJsYCjZKzy5d9zqsOkILBKpqnpVu5FcJlXdYyc-K0TMNsWbi0zsA3F7OBchEfOREZcdwZNfN7Qv4s5pb38ZdPQsfD_wGrvjzgE5FdOOvV7-9gEREI7ruytTawsoHSLMd_lwxIfXBUUJxJSAb13oDKEGWY5N9fVN0B3ayuRtwjZF-nw", + "File": "env", + "SymlinkFile": "", + "Commit": "4bbb59be36182bdab8829924c311a6109bc30748", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/4bbb59be36182bdab8829924c311a6109bc30748/env#L17", + "Entropy": 5.886606, + "Author": "Pablo", + "Email": "pgomezna@gmail.com", + "Date": "2025-04-04T17:32:22Z", + "Message": "API change", + "Tags": [], + "Fingerprint": "4bbb59be36182bdab8829924c311a6109bc30748:env:jwt:17" + }, + { + "RuleID": "jwt", + "Description": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", + "StartLine": 9, + "EndLine": 9, + "StartColumn": 11, + "EndColumn": 522, + "Match": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQzNzA4Nzg1LCJqdGkiOiJlNDlhNjgwMjQ1YzA0ZjI1OTM2YWQxMjgyNWZjYTgyNiIsInVzZXJfaWQiOjc1OTg5Nn0.FkmK4xi260OzsyXqvqOyV6DyzTysqUkBp4Vefm_267VKXgHwuTY_dKgVKjcjhYyDkMQWGDvx0v2NOz8Eamx2RKo81jZtb0vgmHBWwQy9pse8Xj5PJIFrDy0znQvF0gMc5Jwr-ebAkiZizufg7E4a4P4T8sEm2OYauup89769TZDWHcHfE3iBoDWW24J2x5m6PlCJZQM-Ro4DZPhXJezAoCi8pewG56VUp02LsNo_yW1S-vB1n3V6xcaEJjKaYLx8iKuQ4FtgMZB6wcCGD-HD9aZmrFfrQf_wi5qCuIz0hf8gMJC33OwGrlsTDr626Xb_qpbnABRLTvxjY0D0NnLq8A'", + "Secret": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQzNzA4Nzg1LCJqdGkiOiJlNDlhNjgwMjQ1YzA0ZjI1OTM2YWQxMjgyNWZjYTgyNiIsInVzZXJfaWQiOjc1OTg5Nn0.FkmK4xi260OzsyXqvqOyV6DyzTysqUkBp4Vefm_267VKXgHwuTY_dKgVKjcjhYyDkMQWGDvx0v2NOz8Eamx2RKo81jZtb0vgmHBWwQy9pse8Xj5PJIFrDy0znQvF0gMc5Jwr-ebAkiZizufg7E4a4P4T8sEm2OYauup89769TZDWHcHfE3iBoDWW24J2x5m6PlCJZQM-Ro4DZPhXJezAoCi8pewG56VUp02LsNo_yW1S-vB1n3V6xcaEJjKaYLx8iKuQ4FtgMZB6wcCGD-HD9aZmrFfrQf_wi5qCuIz0hf8gMJC33OwGrlsTDr626Xb_qpbnABRLTvxjY0D0NnLq8A", + "File": "utils/members.py", + "SymlinkFile": "", + "Commit": "e13694037199e3bea3de23ae0131e636c62d5aed", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/e13694037199e3bea3de23ae0131e636c62d5aed/utils/members.py#L9", + "Entropy": 5.888001, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-04-04T10:01:44Z", + "Message": "token modifications", + "Tags": [], + "Fingerprint": "e13694037199e3bea3de23ae0131e636c62d5aed:utils/members.py:jwt:9" + }, + { + "RuleID": "github-pat", + "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "StartLine": 24, + "EndLine": 24, + "StartColumn": 20, + "EndColumn": 59, + "Match": "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", + "Secret": "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", + "File": "recovery/github_recovery.py", + "SymlinkFile": "", + "Commit": "4c29834f8242fe60c28182f982df7c9616923cf4", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/4c29834f8242fe60c28182f982df7c9616923cf4/recovery/github_recovery.py#L24", + "Entropy": 4.803056, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-05-27T00:26:14Z", + "Message": "recovry", + "Tags": [], + "Fingerprint": "4c29834f8242fe60c28182f982df7c9616923cf4:recovery/github_recovery.py:github-pat:24" + }, + { + "RuleID": "github-pat", + "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "StartLine": 10, + "EndLine": 10, + "StartColumn": 44, + "EndColumn": 83, + "Match": "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", + "Secret": "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", + "File": "datasources/github_handler.py", + "SymlinkFile": "", + "Commit": "fb5af68da7c610cb306db0db52fdae2473fa7a28", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/fb5af68da7c610cb306db0db52fdae2473fa7a28/datasources/github_handler.py#L10", + "Entropy": 4.8341837, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-26T15:54:51Z", + "Message": "Solving problems with API call to commit stats", + "Tags": [], + "Fingerprint": "fb5af68da7c610cb306db0db52fdae2473fa7a28:datasources/github_handler.py:github-pat:10" + }, + { + "RuleID": "github-pat", + "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "StartLine": 7, + "EndLine": 7, + "StartColumn": 28, + "EndColumn": 67, + "Match": "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", + "Secret": "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", + "File": "datasources/github_handler.py", + "SymlinkFile": "", + "Commit": "429d462f6209cc32022bb0d8ecab3052c16ed1d7", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/429d462f6209cc32022bb0d8ecab3052c16ed1d7/datasources/github_handler.py#L7", + "Entropy": 4.8341837, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-26T13:02:39Z", + "Message": "Commit Stats First implementation", + "Tags": [], + "Fingerprint": "429d462f6209cc32022bb0d8ecab3052c16ed1d7:datasources/github_handler.py:github-pat:7" + }, + { + "RuleID": "github-pat", + "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "StartLine": 7, + "EndLine": 7, + "StartColumn": 72, + "EndColumn": 111, + "Match": "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", + "Secret": "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", + "File": "datasources/github_handler.py", + "SymlinkFile": "", + "Commit": "429d462f6209cc32022bb0d8ecab3052c16ed1d7", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/429d462f6209cc32022bb0d8ecab3052c16ed1d7/datasources/github_handler.py#L7", + "Entropy": 4.8341837, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-26T13:02:39Z", + "Message": "Commit Stats First implementation", + "Tags": [], + "Fingerprint": "429d462f6209cc32022bb0d8ecab3052c16ed1d7:datasources/github_handler.py:github-pat:7" + }, + { + "RuleID": "github-pat", + "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "StartLine": 6, + "EndLine": 6, + "StartColumn": 28, + "EndColumn": 67, + "Match": "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", + "Secret": "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", + "File": "utils/delete_webhooks_github.py", + "SymlinkFile": "", + "Commit": "468587ecc60805fad0da7d4c8201705392cfd188", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/468587ecc60805fad0da7d4c8201705392cfd188/utils/delete_webhooks_github.py#L6", + "Entropy": 4.8341837, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-26T10:05:46Z", + "Message": "Script fot Deletion of Webhooks", + "Tags": [], + "Fingerprint": "468587ecc60805fad0da7d4c8201705392cfd188:utils/delete_webhooks_github.py:github-pat:6" + }, + { + "RuleID": "github-pat", + "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "StartLine": 6, + "EndLine": 6, + "StartColumn": 72, + "EndColumn": 111, + "Match": "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", + "Secret": "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", + "File": "utils/delete_webhooks_github.py", + "SymlinkFile": "", + "Commit": "468587ecc60805fad0da7d4c8201705392cfd188", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/468587ecc60805fad0da7d4c8201705392cfd188/utils/delete_webhooks_github.py#L6", + "Entropy": 4.8341837, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-26T10:05:46Z", + "Message": "Script fot Deletion of Webhooks", + "Tags": [], + "Fingerprint": "468587ecc60805fad0da7d4c8201705392cfd188:utils/delete_webhooks_github.py:github-pat:6" + }, + { + "RuleID": "jwt", + "Description": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", + "StartLine": 10, + "EndLine": 10, + "StartColumn": 27, + "EndColumn": 538, + "Match": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQyOTQxMDg1LCJqdGkiOiJiMzJhZjE1NDU4NzI0OThhOTY2NWJhMWVkOTIxZjMzYSIsInVzZXJfaWQiOjc1OTg5Nn0.H8ug241uZP6q2AbSKSZZbcxRSlRE52wGwok8FfZGhVjnshGqGkbEJnhodESqI94bT6bPCToXxDeLqLEdwQDMz7t5j7vDioxyG8cxF2E2bjLVLt6-wkT6U1mdomyyYvpP78gkmlNPr0a3Jv60MplTgcSnfrQ11N2xYDzf1yNtTJsYCjZKzy5d9zqsOkILBKpqnpVu5FcJlXdYyc-K0TMNsWbi0zsA3F7OBchEfOREZcdwZNfN7Qv4s5pb38ZdPQsfD_wGrvjzgE5FdOOvV7-9gEREI7ruytTawsoHSLMd_lwxIfXBUUJxJSAb13oDKEGWY5N9fVN0B3ayuRtwjZF-nw\"", + "Secret": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQyOTQxMDg1LCJqdGkiOiJiMzJhZjE1NDU4NzI0OThhOTY2NWJhMWVkOTIxZjMzYSIsInVzZXJfaWQiOjc1OTg5Nn0.H8ug241uZP6q2AbSKSZZbcxRSlRE52wGwok8FfZGhVjnshGqGkbEJnhodESqI94bT6bPCToXxDeLqLEdwQDMz7t5j7vDioxyG8cxF2E2bjLVLt6-wkT6U1mdomyyYvpP78gkmlNPr0a3Jv60MplTgcSnfrQ11N2xYDzf1yNtTJsYCjZKzy5d9zqsOkILBKpqnpVu5FcJlXdYyc-K0TMNsWbi0zsA3F7OBchEfOREZcdwZNfN7Qv4s5pb38ZdPQsfD_wGrvjzgE5FdOOvV7-9gEREI7ruytTawsoHSLMd_lwxIfXBUUJxJSAb13oDKEGWY5N9fVN0B3ayuRtwjZF-nw", + "File": "utils/delete_webhooks_taiga.py", + "SymlinkFile": "", + "Commit": "468587ecc60805fad0da7d4c8201705392cfd188", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/468587ecc60805fad0da7d4c8201705392cfd188/utils/delete_webhooks_taiga.py#L10", + "Entropy": 5.886606, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-26T10:05:46Z", + "Message": "Script fot Deletion of Webhooks", + "Tags": [], + "Fingerprint": "468587ecc60805fad0da7d4c8201705392cfd188:utils/delete_webhooks_taiga.py:jwt:10" + }, + { + "RuleID": "jwt", + "Description": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", + "StartLine": 10, + "EndLine": 10, + "StartColumn": 542, + "EndColumn": 1053, + "Match": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQyOTQxMDg1LCJqdGkiOiJiMzJhZjE1NDU4NzI0OThhOTY2NWJhMWVkOTIxZjMzYSIsInVzZXJfaWQiOjc1OTg5Nn0.H8ug241uZP6q2AbSKSZZbcxRSlRE52wGwok8FfZGhVjnshGqGkbEJnhodESqI94bT6bPCToXxDeLqLEdwQDMz7t5j7vDioxyG8cxF2E2bjLVLt6-wkT6U1mdomyyYvpP78gkmlNPr0a3Jv60MplTgcSnfrQ11N2xYDzf1yNtTJsYCjZKzy5d9zqsOkILBKpqnpVu5FcJlXdYyc-K0TMNsWbi0zsA3F7OBchEfOREZcdwZNfN7Qv4s5pb38ZdPQsfD_wGrvjzgE5FdOOvV7-9gEREI7ruytTawsoHSLMd_lwxIfXBUUJxJSAb13oDKEGWY5N9fVN0B3ayuRtwjZF-nw\"", + "Secret": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQyOTQxMDg1LCJqdGkiOiJiMzJhZjE1NDU4NzI0OThhOTY2NWJhMWVkOTIxZjMzYSIsInVzZXJfaWQiOjc1OTg5Nn0.H8ug241uZP6q2AbSKSZZbcxRSlRE52wGwok8FfZGhVjnshGqGkbEJnhodESqI94bT6bPCToXxDeLqLEdwQDMz7t5j7vDioxyG8cxF2E2bjLVLt6-wkT6U1mdomyyYvpP78gkmlNPr0a3Jv60MplTgcSnfrQ11N2xYDzf1yNtTJsYCjZKzy5d9zqsOkILBKpqnpVu5FcJlXdYyc-K0TMNsWbi0zsA3F7OBchEfOREZcdwZNfN7Qv4s5pb38ZdPQsfD_wGrvjzgE5FdOOvV7-9gEREI7ruytTawsoHSLMd_lwxIfXBUUJxJSAb13oDKEGWY5N9fVN0B3ayuRtwjZF-nw", + "File": "utils/delete_webhooks_taiga.py", + "SymlinkFile": "", + "Commit": "468587ecc60805fad0da7d4c8201705392cfd188", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/468587ecc60805fad0da7d4c8201705392cfd188/utils/delete_webhooks_taiga.py#L10", + "Entropy": 5.886606, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-26T10:05:46Z", + "Message": "Script fot Deletion of Webhooks", + "Tags": [], + "Fingerprint": "468587ecc60805fad0da7d4c8201705392cfd188:utils/delete_webhooks_taiga.py:jwt:10" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 56, + "EndLine": 56, + "StartColumn": 6, + "EndColumn": 35, + "Match": "key: ed25519.Ed25519PrivateKey", + "Secret": "ed25519.Ed25519PrivateKey", + "File": "myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py", + "SymlinkFile": "", + "Commit": "13add36a5bbcb3e039f0fea45dc1188ab9747a36", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/13add36a5bbcb3e039f0fea45dc1188ab9747a36/myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py#L56", + "Entropy": 3.8136606, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-24T19:14:14Z", + "Message": "LD_Connect Working", + "Tags": [], + "Fingerprint": "13add36a5bbcb3e039f0fea45dc1188ab9747a36:myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py:generic-api-key:56" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 57, + "EndLine": 57, + "StartColumn": 6, + "EndColumn": 40, + "Match": "key_cls = ed25519.Ed25519PrivateKey", + "Secret": "ed25519.Ed25519PrivateKey", + "File": "myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py", + "SymlinkFile": "", + "Commit": "13add36a5bbcb3e039f0fea45dc1188ab9747a36", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/13add36a5bbcb3e039f0fea45dc1188ab9747a36/myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py#L57", + "Entropy": 3.8136606, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-24T19:14:14Z", + "Message": "LD_Connect Working", + "Tags": [], + "Fingerprint": "13add36a5bbcb3e039f0fea45dc1188ab9747a36:myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py:generic-api-key:57" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 68, + "EndLine": 68, + "StartColumn": 6, + "EndColumn": 31, + "Match": "key: ed448.Ed448PrivateKey", + "Secret": "ed448.Ed448PrivateKey", + "File": "myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py", + "SymlinkFile": "", + "Commit": "13add36a5bbcb3e039f0fea45dc1188ab9747a36", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/13add36a5bbcb3e039f0fea45dc1188ab9747a36/myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py#L68", + "Entropy": 3.5944657, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-24T19:14:14Z", + "Message": "LD_Connect Working", + "Tags": [], + "Fingerprint": "13add36a5bbcb3e039f0fea45dc1188ab9747a36:myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py:generic-api-key:68" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 69, + "EndLine": 69, + "StartColumn": 6, + "EndColumn": 36, + "Match": "key_cls = ed448.Ed448PrivateKey", + "Secret": "ed448.Ed448PrivateKey", + "File": "myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py", + "SymlinkFile": "", + "Commit": "13add36a5bbcb3e039f0fea45dc1188ab9747a36", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/13add36a5bbcb3e039f0fea45dc1188ab9747a36/myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py#L69", + "Entropy": 3.5944657, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-24T19:14:14Z", + "Message": "LD_Connect Working", + "Tags": [], + "Fingerprint": "13add36a5bbcb3e039f0fea45dc1188ab9747a36:myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py:generic-api-key:69" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 43, + "EndLine": 43, + "StartColumn": 10, + "EndColumn": 41, + "Match": "RegEnumKey = win32api.RegEnumKey", + "Secret": "win32api.RegEnumKey", + "File": "myenv/Lib/site-packages/setuptools/_distutils/msvccompiler.py", + "SymlinkFile": "", + "Commit": "13add36a5bbcb3e039f0fea45dc1188ab9747a36", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/13add36a5bbcb3e039f0fea45dc1188ab9747a36/myenv/Lib/site-packages/setuptools/_distutils/msvccompiler.py#L43", + "Entropy": 3.932138, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-03-24T19:14:14Z", + "Message": "LD_Connect Working", + "Tags": [], + "Fingerprint": "13add36a5bbcb3e039f0fea45dc1188ab9747a36:myenv/Lib/site-packages/setuptools/_distutils/msvccompiler.py:generic-api-key:43" + } +] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..960cd8e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.11.0 + hooks: + - id: ruff + args: ["--fix"] + + - repo: https://github.com/gitleaks/gitleaks + rev: v8.30.0 + hooks: + - id: gitleaks + args: ["--redact=0", "--baseline-path", ".gitleaks-baseline.json", "--exit-code", "1"] diff --git a/.sonarcloud.properties b/.sonarcloud.properties new file mode 100644 index 0000000..8c0335f --- /dev/null +++ b/.sonarcloud.properties @@ -0,0 +1 @@ +sonar.exclusions=deprecated/**,.gitleaks-baseline.json,tests/**,deprecated/** diff --git a/README.md b/README.md index 152fe86..d9c04dd 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ # LD Connect – Event Ingestion Service -**LD Connect** is the entry point of the Learning Dashboard pipeline. +**LD Connect** is the entry point of the Learning Dashboard pipeline. Whenever a student pushes to **GitHub**, edits a task on **Taiga**, or logs effort in **Google Sheets**, the event first reaches this service. LD Connect -1. **Authenticates** the webhook (HMAC signatures) -2. **Normalises** the payload to a common schema -3. **Persists** it in MongoDB (idempotent upserts) +1. **Authenticates** the webhook (HMAC signatures) +2. **Normalises** the payload to a common schema +3. **Persists** it in MongoDB (idempotent upserts) 4. **Notifies** LD Eval so metrics are recalculated in near real‑time --- @@ -27,10 +27,10 @@ Whenever a student pushes to **GitHub**, edits a task on **Taiga**, or logs effo ```text ┌──────────────┐ Webhook ┌──────────────┐ │ GitHub │───POST────▶ │ │ -└──────────────┘ │ │ -┌──────────────┐ │ │ +└──────────────┘ │ │ +┌──────────────┐ │ │ │ Taiga │───POST────▶ │ LD Connect │──┐ POST /api/event -└──────────────┘ │ │ │ +└──────────────┘ │ │ │ ┌──────────────┐ │ │ │ │ GoogleSheet │───POST────▶│ │ │ (notify) └──────────────┘ └──────────────┘ │ @@ -90,8 +90,8 @@ curl -X POST "http://127.0.0.1:5000/webhook/github?ping=1" docker compose up -d --build ld_connect ``` -* Exposes the service on port **5000** inside the container -* Behind Nginx / Traefik, route +* Exposes the service on port **5000** inside the container +* Behind Nginx / Traefik, route `https:///webhook/{github|taiga|excel}` → `ld_connect:5000` --- @@ -115,12 +115,12 @@ Store them in `.env` (already referenced in `docker-compose.yml`). ### `POST /webhook/github` -Receives any GitHub event subscribed in the repo webhook. +Receives any GitHub event subscribed in the repo webhook. Requires headers `X-Hub-Signature` **and** `X-Hub-Signature-256`. ### `POST /webhook/taiga` -Receives Taiga events. +Receives Taiga events. Requires header `X-Taiga-Webhook-Signature`. ### `POST /webhook/excel` @@ -142,9 +142,44 @@ All endpoints return **`200 OK`** immediately; heavy work continues asynchronou ```bash pytest # unit tests -locust -f tests/ # stress tests (replay real‑world payloads) ``` +If you just cloned the repository, set up `pre-commit` once before you start coding. +It will automatically run checks every time you commit, so common issues are caught early. + +### First-time setup (after cloning) + +```bash +# 1) create and activate a virtual environment (if you did not do it yet) +python -m venv .venv +source .venv/bin/activate + +# 2) install project dependencies +pip install -r requirements.txt + +# 3) install pre-commit in your environment +pip install pre-commit + +# 4) install git hooks for this repository (one-time) +pre-commit install + +# 5) optional: run checks on all files now +pre-commit run --all-files +``` + +### Why this helps + +- `ruff` catches Python style/quality issues and can auto-fix many of them. +- `gitleaks` helps prevent committing secrets (tokens, passwords, keys). +- Because hooks run before each commit, problems are found locally instead of failing later in CI. + +Configured hooks: + +| Hook | Purpose | +| --- | --- | +| `ruff` | Python linting (with autofix) | +| `gitleaks` | Detect hardcoded secrets | + --- ## FAQs diff --git a/app.py b/app.py index c204a75..dbbd008 100644 --- a/app.py +++ b/app.py @@ -4,10 +4,12 @@ from routes.excel_routes import excel_bp from config.logger_config import setup_logging import logging +import os setup_logging() logger = logging.getLogger(__name__) + def create_app(): app = Flask(__name__) @@ -18,6 +20,11 @@ def create_app(): logger.info("Flask created and Blueprints registered successfully.") return app + if __name__ == "__main__": app = create_app() - app.run(debug=True, host="127.0.0.1", port=5000) + app.run( + debug=os.getenv("FLASK_DEBUG", "false").lower() == "true", + host="127.0.0.1", + port=5000, + ) diff --git a/config/credentials_loader.py b/config/credentials_loader.py index 43ae183..0f6c971 100644 --- a/config/credentials_loader.py +++ b/config/credentials_loader.py @@ -1,7 +1,8 @@ -import json, os +import json +import os from typing import Optional -CONFIG_FILE = os.getenv("CREDENTIALS_FILE", - "config_files/credentials_config.json") + +CONFIG_FILE = os.getenv("CREDENTIALS_FILE", "config_files/credentials_config.json") def load(): @@ -16,9 +17,7 @@ def resolve(prj: str, field: str) -> Optional[str]: """ cfg = load() for course, props in cfg.items(): - if prj in props["teams"]: + if prj in props["teams"]: return props.get(field) raise KeyError(f"Project {prj!r} not found in {CONFIG_FILE}") - - diff --git a/config/logger_config.py b/config/logger_config.py index f21c47f..e12524e 100644 --- a/config/logger_config.py +++ b/config/logger_config.py @@ -4,6 +4,7 @@ _DEFAULT_FMT = "%(asctime)s [%(levelname)s] %(name)s: %(message)s" + def setup_logging() -> None: """ Configure the logger. diff --git a/config/settings.py b/config/settings.py index 33eb57e..ffc40ca 100644 --- a/config/settings.py +++ b/config/settings.py @@ -6,7 +6,8 @@ BASE_DIR = Path(__file__).resolve().parent.parent # Load environment variables from the .env file -load_dotenv(BASE_DIR / '.env') +load_dotenv(BASE_DIR / ".env") + def _require_env(name: str) -> str: value = os.getenv(name) @@ -17,18 +18,21 @@ def _require_env(name: str) -> str: ) return value -#Mongo database settings -MONGO_HOST = os.getenv("MONGO_HOST", "mongodb") -MONGO_PORT = os.getenv("MONGO_PORT", "27017") -MONGO_DB = os.getenv("MONGO_DB", "mongo") -MONGO_USER = os.getenv("MONGO_USER", "") -MONGO_PASS = os.getenv("MONGO_PASS", "") -MONGO_AUTHSRC = os.getenv("MONGO_AUTHSRC", MONGO_DB) + +# Mongo database settings +MONGO_HOST = os.getenv("MONGO_HOST", "mongodb") +MONGO_PORT = os.getenv("MONGO_PORT", "27017") +MONGO_DB = os.getenv("MONGO_DB", "mongo") +MONGO_USER = os.getenv("MONGO_USER", "") +MONGO_PASS = os.getenv("MONGO_PASS", "") +MONGO_AUTHSRC = os.getenv("MONGO_AUTHSRC", MONGO_DB) if MONGO_USER and MONGO_PASS: - MONGO_URI = (f"mongodb://{MONGO_USER}:{MONGO_PASS}" - f"@{MONGO_HOST}:{MONGO_PORT}/{MONGO_DB}" - f"?authSource={MONGO_AUTHSRC}") + MONGO_URI = ( + f"mongodb://{MONGO_USER}:{MONGO_PASS}" + f"@{MONGO_HOST}:{MONGO_PORT}/{MONGO_DB}" + f"?authSource={MONGO_AUTHSRC}" + ) else: MONGO_URI = f"mongodb://{MONGO_HOST}:{MONGO_PORT}/{MONGO_DB}" @@ -42,9 +46,9 @@ def _require_env(name: str) -> str: TAIGA_TOKEN = os.getenv("TAIGA_TOKEN", "") TAIGA_SIGNATURE_KEY = _require_env("TAIGA_SIGNATURE_KEY") TAIGA_USERNAME = _require_env("TAIGA_USERNAME") -TAIGA_PASSWORD= _require_env("TAIGA_PASSWORD") +TAIGA_PASSWORD = _require_env("TAIGA_PASSWORD") # Load the webhook URLs from the environment to enable the deletion of webhooks WEBHOOK_URL_GITHUB = os.getenv("WEBHOOK_URL_GITHUB", "") -WEBHOOK_URL_TAIGA = os.getenv("WEBHOOK_URL_TAIGA", "") \ No newline at end of file +WEBHOOK_URL_TAIGA = os.getenv("WEBHOOK_URL_TAIGA", "") diff --git a/datasources/excel_handler.py b/datasources/excel_handler.py index 903a17c..153034c 100644 --- a/datasources/excel_handler.py +++ b/datasources/excel_handler.py @@ -1,6 +1,3 @@ -from datetime import datetime - - # List of activity types extracted from the Excel sheet ACTIVITY_TYPES = [ "Reunió d'equip", @@ -10,38 +7,37 @@ "Desenvolupament", "Gestió de projecte", "Documentació", - "Presentació" + "Presentació", ] + def parse_excel_event(raw_payload: dict, prj, quality_model) -> dict: - ''' + """ Function to parse the payload from the Excel webhook and return a dict with the data - ''' + """ # Receive the paraemters from the API query quality_model = quality_model - + # Get the values from the payload - ts = raw_payload.get("timestamp") - iteration= raw_payload.get("iteration") - date = raw_payload.get("date") + ts = raw_payload.get("timestamp") + iteration = raw_payload.get("iteration") + date = raw_payload.get("date") duration = raw_payload.get("duration") activity = raw_payload.get("activity") - comment = raw_payload.get("comment") - epic = raw_payload.get("epic") - members = raw_payload.get("members", []) - hours = raw_payload.get("memberHours", []) - config = raw_payload.get("configRange", []) + comment = raw_payload.get("comment") + epic = raw_payload.get("epic") + members = raw_payload.get("members", []) + hours = raw_payload.get("memberHours", []) + config = raw_payload.get("configRange", []) - - # Clean the members list, sometimes there are empty strings + # Clean the members list, sometimes there are empty strings members_clean = [m.strip() for m in members if isinstance(m, str) and m.strip()] # Pair the members with their hours, if there are more members than hours, we will ignore the extra members - hours_clean = hours[:len(members_clean)] + hours_clean = hours[: len(members_clean)] # Create a dictionary with the members and their hours members_dict = { - f"hours_{member}": hours_clean[idx] - for idx, member in enumerate(members_clean) + f"hours_{member}": hours_clean[idx] for idx, member in enumerate(members_clean) } # Map the activity types to the configRange values @@ -54,24 +50,20 @@ def parse_excel_event(raw_payload: dict, prj, quality_model) -> dict: # Sum the hours for each activity type total_hours = sum(activity_hours.values()) - - # Finally, return a dict containing the full structure return { - "timestamp": ts, - "team": prj, - "quality_model": quality_model, - "iteration": iteration, - "activity_date": date, - "duration_h": duration, - "activity_type": activity, - "comment": comment, - "epic": epic, - "members": members_clean, + "timestamp": ts, + "team": prj, + "quality_model": quality_model, + "iteration": iteration, + "activity_date": date, + "duration_h": duration, + "activity_type": activity, + "comment": comment, + "epic": epic, + "members": members_clean, **members_dict, **activity_hours, - "total_hours": total_hours + "total_hours": total_hours, } - - diff --git a/datasources/github_handler.py b/datasources/github_handler.py index 306996a..72b35f6 100644 --- a/datasources/github_handler.py +++ b/datasources/github_handler.py @@ -1,9 +1,9 @@ from typing import Dict import re from datasources.requests.github_api_call import fetch_commit_stats -from config.credentials_loader import resolve from utils.datetime_utils import to_madrid_local + def parse_github_event(raw_payload: Dict, prj: str) -> Dict: """ Parse a GitHub event payload into a more detailed structure. @@ -14,19 +14,20 @@ def parse_github_event(raw_payload: Dict, prj: str) -> Dict: if event_type == "push": return parse_github_push_event(raw_payload, prj) elif event_type == "issues": - return parse_github_issue_event(raw_payload, prj) + return parse_github_issue_event(raw_payload) elif event_type == "pull_request": return parse_github_pullrequest_event(raw_payload, prj) else: return {"event": event_type, "ignored": True} - def parse_github_push_event(raw_payload: Dict, prj: str) -> Dict: - ''' + """ Function to parse a GitHub push event payload. - ''' - event_type = "commit" # The event type is "push" but we will call it "commit" in our system + """ + event_type = ( + "commit" # The event type is "push" but we will call it "commit" in our system + ) # Retrieve the team name and repo name from the payload team_name = raw_payload.get("organization", {}).get("login", "UnknownTeam") @@ -39,9 +40,9 @@ def parse_github_push_event(raw_payload: Dict, prj: str) -> Dict: "login": sender.get("login", ""), "url": sender.get("url", ""), "type": sender.get("type", ""), - "site_admin": sender.get("site_admin", False) + "site_admin": sender.get("site_admin", False), } - + commits_info = [] # The push event typically has a "commits" array for c in raw_payload.get("commits", []): @@ -49,21 +50,20 @@ def parse_github_push_event(raw_payload: Dict, prj: str) -> Dict: commit_sha = c.get("id") commit_url = c.get("url", "") message = c.get("message", "") - #get timestamp of comit in the hour in spain - date= to_madrid_local(c.get("timestamp")) + # get timestamp of comit in the hour in spain + date = to_madrid_local(c.get("timestamp")) # Built author information author_login = c.get("author", {}).get("username", "") - author_name = c.get("author", {}).get("name", "") + author_name = c.get("author", {}).get("name", "") author_email = c.get("author", {}).get("email", "") # Compute message stats message_char_count = len(message) message_word_count = len(message.split()) - # Check if the commit message contains a task reference, it can be in english or catalan, ¿spanish¿ (e.g., "task #123") - pattern = r'(?i)\b(?:task|tasca)\b(?:\s*#?\s*(\d+))?' + pattern = r"(?i)\b(?:task|tasca)\b(?:\s*#?\s*(\d+))?" match = re.search(pattern, message) if match: task_is_written = True @@ -76,8 +76,8 @@ def parse_github_push_event(raw_payload: Dict, prj: str) -> Dict: verified = "false" verified_reason = "unsigned" - - commit_stats = fetch_commit_stats(repo_name, commit_sha, prj) + + commit_stats = fetch_commit_stats(repo_name, commit_sha, prj) # Build a final doc for this commit. commit_doc = { @@ -93,11 +93,11 @@ def parse_github_push_event(raw_payload: Dict, prj: str) -> Dict: "message": message, "message_char_count": message_char_count, "message_word_count": message_word_count, - "task_is_written": task_is_written, - "task_reference": task_reference, + "task_is_written": task_is_written, + "task_reference": task_reference, "verified": verified, "verified_reason": verified_reason, - "stats": commit_stats + "stats": commit_stats, } commits_info.append(commit_doc) @@ -108,20 +108,18 @@ def parse_github_push_event(raw_payload: Dict, prj: str) -> Dict: "repo_name": repo_name, "team_name": team_name, "sender_info": sender_info, - "commits": commits_info + "commits": commits_info, } - - -def parse_github_issue_event(raw_payload: Dict, prj: str) -> Dict: - ''' +def parse_github_issue_event(raw_payload: Dict) -> Dict: + """ Function to parse a GitHub issue event payload. - ''' + """ # Retrieve the event information from the payload - action = raw_payload.get("action", "unknown-action")# e.g. "issue_opened" - event_type = "issue" - + action = raw_payload.get("action", "unknown-action") # e.g. "issue_opened" + event_type = "issue" + # Retrieve the team name and repo name from the payload team_name = raw_payload.get("organization", {}).get("login", "UnknownTeam") repo_name = raw_payload["repository"].get("full_name", "unknown-repo") @@ -133,16 +131,16 @@ def parse_github_issue_event(raw_payload: Dict, prj: str) -> Dict: "login": sender.get("login", ""), "url": sender.get("url", ""), "type": sender.get("type", ""), - "site_admin": sender.get("site_admin", False) + "site_admin": sender.get("site_admin", False), } # The "issue" object is typically raw_payload["issue"] issue_data = raw_payload.get("issue", {}) issue_number = issue_data.get("number", 0) - issue_title = issue_data.get("title", "") - issue_state = issue_data.get("state", "") - issue_body = issue_data.get("body", "") - issue_user = issue_data.get("user", {}) + issue_title = issue_data.get("title", "") + issue_state = issue_data.get("state", "") + issue_body = issue_data.get("body", "") + issue_user = issue_data.get("user", {}) issue_user_login = issue_user.get("login", "") issue_user_id = issue_user.get("id", "") @@ -152,37 +150,31 @@ def parse_github_issue_event(raw_payload: Dict, prj: str) -> Dict: "title": issue_title, "state": issue_state, "body": issue_body, - "user": { - "login": issue_user_login, - "id": issue_user_id - } + "user": {"login": issue_user_login, "id": issue_user_id}, } # Finally, return a dict containing the full structure return { - "event": event_type, - "action": action, + "event": event_type, + "action": action, "repo_name": repo_name, "team_name": team_name, "sender_info": sender_info, - "issue": issue_obj + "issue": issue_obj, } - def parse_github_pullrequest_event(raw_payload: Dict, prj: str) -> Dict: - ''' + """ Function to parse a GitHub pull request event payload. - ''' - + """ + action = raw_payload.get("action") - - if action != "closed": # we only want to process closed pull requests - return {"event": "pull_request", "ignored": True} - - event_type = "pull_request" # The event type is "pull request" but we will call it "commit" in our system + if action != "closed": # we only want to process closed pull requests + return {"event": "pull_request", "ignored": True} + event_type = "pull_request" # The event type is "pull request" but we will call it "commit" in our system # The 'sender' object is at the top level sender = raw_payload.get("sender", {}) @@ -191,31 +183,29 @@ def parse_github_pullrequest_event(raw_payload: Dict, prj: str) -> Dict: "login": sender.get("login", ""), "url": sender.get("url", ""), "type": sender.get("type", ""), - "site_admin": sender.get("site_admin", False) + "site_admin": sender.get("site_admin", False), } - + # Get the information about the pull request pr_info = raw_payload.get("pull_request", {}) - + pr_number = pr_info.get("number", 0) - pr_title = pr_info.get("title", "") + pr_title = pr_info.get("title", "") pr_created_at = to_madrid_local(pr_info.get("created_at", "")) - pr_closed_at = to_madrid_local(pr_info.get("closed_at", "")) + pr_closed_at = to_madrid_local(pr_info.get("closed_at", "")) pr_merged_at = pr_info.get("merged", False) merged_by = pr_info.get("merged_by", {}).get("login", "") - - - pr_assignee = (pr_info.get("assignee") or {}).get("login"), + + pr_assignee = ((pr_info.get("assignee") or {}).get("login"),) pr_reviewers = [r["login"] for r in pr_info.get("requested_reviewers", [])] - - + team_name = raw_payload.get("organization", {}).get("login", "UnknownTeam") repo_name = raw_payload["repository"].get("full_name", "unknown-repo") - - # Build a final doc for this commit. + + # Build a final doc for this commit. pr_doc = { "event": event_type, - "action": action, # always "closed" + "action": action, # always "closed" "pr_number": pr_number, "title": pr_title, "created_at": pr_created_at, @@ -227,8 +217,7 @@ def parse_github_pullrequest_event(raw_payload: Dict, prj: str) -> Dict: "repo_name": repo_name, "team_name": team_name, "sender_info": sender_info, - } - + } # Finally, return a dict containing the full structure - return pr_doc \ No newline at end of file + return pr_doc diff --git a/datasources/requests/github_api_call.py b/datasources/requests/github_api_call.py index 1827cc6..d26a145 100644 --- a/datasources/requests/github_api_call.py +++ b/datasources/requests/github_api_call.py @@ -8,19 +8,20 @@ logger = logging.getLogger(__name__) -def fetch_commit_stats(repo_full_name: str, commit_sha: str, prj: str) -> Dict[str, int]: +def fetch_commit_stats( + repo_full_name: str, commit_sha: str, prj: str +) -> Dict[str, int]: """ Gets 'additions', 'deletions' y 'total' of a commit using GitHub's REST API v3. """ - + token = resolve(prj, "github_token") - + headers = { "Accept": "application/vnd.github.v3+json", - "Authorization": f"token {token}" + "Authorization": f"token {token}", } - url = f"{GITHUB_API_URL}/repos/{repo_full_name}/commits/{commit_sha}" try: @@ -30,8 +31,10 @@ def fetch_commit_stats(repo_full_name: str, commit_sha: str, prj: str) -> Dict[s return { "total": stats.get("total", 0), "additions": stats.get("additions", 0), - "deletions": stats.get("deletions", 0) + "deletions": stats.get("deletions", 0), } except Exception as exc: - logger.error("Error fetching commit stats for %s/%s: %s", repo_full_name, commit_sha, exc) + logger.error( + "Error fetching commit stats for %s/%s: %s", repo_full_name, commit_sha, exc + ) return {"total": 0, "additions": 0, "deletions": 0} diff --git a/datasources/requests/taiga_api_call.py b/datasources/requests/taiga_api_call.py index 1d9f603..4d3c9f9 100644 --- a/datasources/requests/taiga_api_call.py +++ b/datasources/requests/taiga_api_call.py @@ -1,6 +1,6 @@ import logging import requests -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from utils.taiga_token.taiga_auth import get_taiga_token from config.credentials_loader import resolve @@ -8,62 +8,72 @@ logger = logging.getLogger(__name__) -_CACHE = {} # key = (project_id, milestone_id) -> (timestamp, stats) -TTL = timedelta(minutes=1) # Cache time-to-live, set to 5 minutes. Means that if the same request is made within 5 minutes, it will return the cached result instead of making a new API call. +_CACHE = {} # key = (project_id, milestone_id) -> (timestamp, stats) +TTL = timedelta( + minutes=1 +) # Cache time-to-live, set to 5 minutes. Means that if the same request is made within 5 minutes, it will return the cached result instead of making a new API call. + def milestone_stats(project_id: str, milestone_id: str, prj: str): - ''' - Fetches the statistics of a milestone in a Taiga project. + """ + Fetches the statistics of a milestone in a Taiga project. Uses caching to avoid frequent API calls to get the taiga token. It refreshes the cache every 5 minutes. - ''' + """ if not project_id or not milestone_id: return {} key = (project_id, milestone_id) - now = datetime.utcnow() - if key in _CACHE and now - _CACHE[key][0]< TTL: + now = datetime.now(timezone.utc) + if key in _CACHE and now - _CACHE[key][0] < TTL: return _CACHE[key][1] user = resolve(prj, "taiga_user") - psw = resolve(prj, "taiga_password") - logger.debug("Resolving Taiga credentials for project %s: user=%s, password=%s", prj, "****" if user else None, "****" if psw else None) + psw = resolve(prj, "taiga_password") + logger.debug( + "Resolving Taiga credentials for project %s: user=%s, password=%s", + prj, + "****" if user else None, + "****" if psw else None, + ) if user and psw: token = get_taiga_token(user, psw) headers = {"Authorization": f"Bearer {token}"} - logger.debug("Using Taiga credentials for project:", prj) + logger.debug("Using Taiga credentials for project %s", prj) else: headers = {} logger.info("Using Taiga tunnel without authentication for project: %s", prj) - + url = f"{TAIGA_API_URL}/milestones/{milestone_id}/stats" logger.debug("Fetching Taiga milestone stats from URL: %s", url) - r = requests.get(url, params={"project": project_id}, headers=headers, timeout=(1, 5)) - + r = requests.get( + url, params={"project": project_id}, headers=headers, timeout=(1, 5) + ) + try: r.raise_for_status() except requests.exceptions.HTTPError as e: print(f"Warning: Failed to fetch milestone stats (status {r.status_code}): {e}") # Return empty stats if we can't access the milestone stats = { - "milestone_total_points" : 0, - "milestone_closed_points" : 0, - "milestone_total_userstories" : 0, + "milestone_total_points": 0, + "milestone_closed_points": 0, + "milestone_total_userstories": 0, "milestone_completed_userstories": 0, - "milestone_total_tasks" : 0, - "milestone_completed_tasks" : 0, + "milestone_total_tasks": 0, + "milestone_completed_tasks": 0, } _CACHE[key] = (now, stats) return stats - - js = r.json() + + js = r.json() stats = { - "milestone_total_points" : sum(js.get("total_points", {}).values()), - "milestone_closed_points" : sum(js.get("completed_points", 0)), - "milestone_total_userstories" : js.get("total_userstories", 0), + "milestone_total_points": sum(js.get("total_points", {}).values()), + "milestone_closed_points": sum(js.get("completed_points", 0)), + "milestone_total_userstories": js.get("total_userstories", 0), "milestone_completed_userstories": js.get("completed_userstories", 0), - "milestone_total_tasks" : js.get("total_tasks", 0), - "milestone_completed_tasks" : js.get("completed_tasks", 0), + "milestone_total_tasks": js.get("total_tasks", 0), + "milestone_completed_tasks": js.get("completed_tasks", 0), } _CACHE[key] = (now, stats) - return stats \ No newline at end of file + return stats diff --git a/datasources/taiga_handler.py b/datasources/taiga_handler.py index 43d3dc3..486cd16 100644 --- a/datasources/taiga_handler.py +++ b/datasources/taiga_handler.py @@ -1,7 +1,7 @@ from typing import Dict import re -from datasources.requests.taiga_api_call import milestone_stats +from datasources.requests.taiga_api_call import milestone_stats from utils.datetime_utils import to_madrid_local @@ -12,8 +12,8 @@ def parse_taiga_event(raw_payload: Dict, prj: str) -> Dict: We can handle "issues", "epics", "tasks" and "userstories" events. """ event_type = raw_payload.get("type") - if event_type == "issue": - return parse_taiga_issue_event(raw_payload, prj) + if event_type == "issue": + return parse_taiga_issue_event(raw_payload) elif event_type == "epic": return parse_taiga_epic_event(raw_payload, prj) elif event_type == "task": @@ -23,42 +23,43 @@ def parse_taiga_event(raw_payload: Dict, prj: str) -> Dict: elif event_type == "relateduserstory": return parse_taiga_related_userstory_event(raw_payload, prj) else: - return { - "event": event_type, - "error": "Unsupported event type" - } + return {"event": event_type, "error": "Unsupported event type"} - - -def parse_taiga_issue_event(raw_payload: Dict, prj: str) -> Dict: - ''' +def parse_taiga_issue_event(raw_payload: Dict) -> Dict: + """ Function to parse a taiga issue event payload. - ''' + """ # Extract the relevant fields from the raw payload - project_id= raw_payload.get("data",{}).get("project",{}).get("id", "") - issue_id=raw_payload.get("data",{}).get("id", "") - team_name = raw_payload.get("data",{}).get("project", {}).get("name", "") - event_type= raw_payload.get("type","") - action_type= raw_payload.get("action","") - subject = raw_payload.get("data",{}).get("subject", "") - due_date = to_madrid_local(raw_payload.get("data",{}).get("due_date", "")) - description = raw_payload.get("data", {}).get("description", "") + project_id = raw_payload.get("data", {}).get("project", {}).get("id", "") + issue_id = raw_payload.get("data", {}).get("id", "") + team_name = raw_payload.get("data", {}).get("project", {}).get("name", "") + event_type = raw_payload.get("type", "") + action_type = raw_payload.get("action", "") + subject = raw_payload.get("data", {}).get("subject", "") + due_date = to_madrid_local(raw_payload.get("data", {}).get("due_date", "")) + description = raw_payload.get("data", {}).get("description", "") severity = raw_payload.get("data", {}).get("severity", {}).get("name", "") status = raw_payload.get("data", {}).get("status", {}).get("name", "") priority = raw_payload.get("data", {}).get("priority", {}).get("name", "") - type = raw_payload.get("data", {}).get("type", {}).get("name", "") + type = raw_payload.get("data", {}).get("type", {}).get("name", "") is_closed = raw_payload.get("is_closed", False) - modified_date = to_madrid_local(raw_payload.get("data", {}).get("modified_date", "")) + modified_date = to_madrid_local( + raw_payload.get("data", {}).get("modified_date", "") + ) created_date = to_madrid_local(raw_payload.get("data", {}).get("created_date", "")) - finished_date = to_madrid_local(raw_payload.get("data", {}).get("finished_date", "")) + finished_date = to_madrid_local( + raw_payload.get("data", {}).get("finished_date", "") + ) assigned_by = raw_payload.get("by", {}).get("username", "") - #There are cases where the assigned_to field is empty, and if we request it aniways it will throw an error, so we need to check if it exists - if raw_payload.get("data", {}).get("assigned_to", {}) != None: - assigned_to = raw_payload.get("data", {}).get("assigned_to", {}).get("username", "") + # There are cases where the assigned_to field is empty, and if we request it aniways it will throw an error, so we need to check if it exists + if raw_payload.get("data", {}).get("assigned_to", {}) is not None: + assigned_to = ( + raw_payload.get("data", {}).get("assigned_to", {}).get("username", "") + ) else: assigned_to = None - + # Create a dictionary with all the attributes of the issue doc = { "project_id": project_id, @@ -78,32 +79,32 @@ def parse_taiga_issue_event(raw_payload: Dict, prj: str) -> Dict: "created_date": created_date, "finished_date": finished_date, "assigned_by": assigned_by, - "assigned_to": assigned_to + "assigned_to": assigned_to, } # Return the parsed issue data as a dictionary return doc - - -def parse_taiga_epic_event(raw_payload: Dict, prj: str) -> Dict: - ''' +def parse_taiga_epic_event(raw_payload: Dict, prj: str) -> Dict: + """ Function to parse a taiga epic event payload. - ''' + """ # Extract the relevant fields from the raw payload - epic_id= raw_payload.get("data",{}).get("id", "") - team_name = raw_payload.get("data",{}).get("project", {}).get("name", "") - event_type= raw_payload.get("type","") - action_type= raw_payload.get("action","") - subject = raw_payload.get("data",{}).get("subject", "") + epic_id = raw_payload.get("data", {}).get("id", "") + team_name = raw_payload.get("data", {}).get("project", {}).get("name", "") + event_type = raw_payload.get("type", "") + action_type = raw_payload.get("action", "") + subject = raw_payload.get("data", {}).get("subject", "") status = raw_payload.get("data", {}).get("status", {}).get("name", "") is_closed = raw_payload.get("is_closed", False) - modified_date = to_madrid_local(raw_payload.get("data", {}).get("modified_date", "")) + modified_date = to_madrid_local( + raw_payload.get("data", {}).get("modified_date", "") + ) created_date = to_madrid_local(raw_payload.get("data", {}).get("created_date", "")) assigned_by = raw_payload.get("by", {}).get("username", "") - #We are going to use this project_id to delete the webhooks with the TAIGA API - project_id= raw_payload.get("data",{}).get("project",{}).get("id", "") - + # We are going to use this project_id to delete the webhooks with the TAIGA API + project_id = raw_payload.get("data", {}).get("project", {}).get("id", "") + # Create a dictionary with all the attributes of the epic doc = { "epic_id": epic_id, @@ -116,58 +117,76 @@ def parse_taiga_epic_event(raw_payload: Dict, prj: str) -> Dict: "status": status, "modified_date": modified_date, "created_date": created_date, - "project_id": project_id - + "project_id": project_id, } # Return the parsed epic data as a dictionary return doc - - -def parse_taiga_task_event(raw_payload: Dict, prj: str) -> Dict: - ''' +def parse_taiga_task_event(raw_payload: Dict, prj: str) -> Dict: + """ Function to parse a taiga task event payload. - ''' + """ # Extract the relevant fields from the raw payload - project_id= raw_payload.get("data",{}).get("project",{}).get("id", "") - team_name = raw_payload.get("data",{}).get("project", {}).get("name", "") - event_type= raw_payload.get("type","") - action_type= raw_payload.get("action","") - task_id= raw_payload.get("data",{}).get("id", "") - subject = raw_payload.get("data",{}).get("subject", "") - userstory_id= raw_payload.get("data",{}).get("user_story",{}).get("id", "") - userstory_is_closed= raw_payload.get("data",{}).get("user_story",{}).get("is_closed", "") + project_id = raw_payload.get("data", {}).get("project", {}).get("id", "") + team_name = raw_payload.get("data", {}).get("project", {}).get("name", "") + event_type = raw_payload.get("type", "") + action_type = raw_payload.get("action", "") + task_id = raw_payload.get("data", {}).get("id", "") + subject = raw_payload.get("data", {}).get("subject", "") + userstory_id = raw_payload.get("data", {}).get("user_story", {}).get("id", "") + userstory_is_closed = ( + raw_payload.get("data", {}).get("user_story", {}).get("is_closed", "") + ) is_closed = raw_payload.get("data", {}).get("status", {}).get("is_closed", "") status = raw_payload.get("data", {}).get("status", {}).get("name", "") created_date = to_madrid_local(raw_payload.get("data", {}).get("created_date", "")) - modified_date = to_madrid_local(raw_payload.get("data", {}).get("modified_date", "")) - finished_date = to_madrid_local(raw_payload.get("data", {}).get("finished_date", "")) - reference=raw_payload.get("data",{}).get("ref", "") - milestone_id=raw_payload.get("data",{}).get("milestone",{}).get("id", "") - milestone_name=raw_payload.get("data",{}).get("milestone",{}).get("name", "") - milestone_closed=raw_payload.get("data",{}).get("milestone",{}).get("closed", "") - milestone_created_date=raw_payload.get("data",{}).get("milestone",{}).get("created_date", "") - milestone_created_date = to_madrid_local(milestone_created_date) if milestone_created_date else "" - milestone_modified_date=raw_payload.get("data",{}).get("milestone",{}).get("modified_date", "") - milestone_modified_date = to_madrid_local(milestone_modified_date) if milestone_modified_date else "" - estimated_start=to_madrid_local(raw_payload.get("data",{}).get("milestone",{}).get("estimated_start", "")) - estimated_finish=to_madrid_local(raw_payload.get("data",{}).get("milestone",{}).get("estimated_finish", "")) + modified_date = to_madrid_local( + raw_payload.get("data", {}).get("modified_date", "") + ) + finished_date = to_madrid_local( + raw_payload.get("data", {}).get("finished_date", "") + ) + reference = raw_payload.get("data", {}).get("ref", "") + milestone_id = raw_payload.get("data", {}).get("milestone", {}).get("id", "") + milestone_name = raw_payload.get("data", {}).get("milestone", {}).get("name", "") + milestone_closed = ( + raw_payload.get("data", {}).get("milestone", {}).get("closed", "") + ) + milestone_created_date = ( + raw_payload.get("data", {}).get("milestone", {}).get("created_date", "") + ) + milestone_created_date = ( + to_madrid_local(milestone_created_date) if milestone_created_date else "" + ) + milestone_modified_date = ( + raw_payload.get("data", {}).get("milestone", {}).get("modified_date", "") + ) + milestone_modified_date = ( + to_madrid_local(milestone_modified_date) if milestone_modified_date else "" + ) + estimated_start = to_madrid_local( + raw_payload.get("data", {}).get("milestone", {}).get("estimated_start", "") + ) + estimated_finish = to_madrid_local( + raw_payload.get("data", {}).get("milestone", {}).get("estimated_finish", "") + ) assigned_by = raw_payload.get("by", {}).get("username", "") - - + milestone_data = milestone_stats(project_id, milestone_id, prj) - #If someone defines a new metric, if it isnt listed in the handler, we wont get it. To solve we can get all the custom attributes as an object and store it in mongo + # If someone defines a new metric, if it isnt listed in the handler, we wont get it. To solve we can get all the custom attributes as an object and store it in mongo custom_attributes = raw_payload.get("data", {}).get("custom_attributes_values", {}) if custom_attributes is None: custom_attributes = {} - - #There are cases where the assigned_to field is empty, and if we request it aniways it will throw an error, so we need to check if it exists - if raw_payload.get("data", {}).get("assigned_to", {}) != None: - assigned_to = raw_payload.get("data", {}).get("assigned_to", {}).get("username", "") + + # There are cases where the assigned_to field is empty, and if we request it aniways it will throw an error, so we need to check if it exists + if raw_payload.get("data", {}).get("assigned_to", {}) is not None: + assigned_to = ( + raw_payload.get("data", {}).get("assigned_to", {}).get("username", "") + ) else: assigned_to = None - + # Create a dictionary with all the attributes of the task doc = { "project_id": project_id, @@ -193,83 +212,90 @@ def parse_taiga_task_event(raw_payload: Dict, prj: str) -> Dict: "milestone_modified_date": milestone_modified_date, "estimated_start": estimated_start, "estimated_finish": estimated_finish, - - #We can get all the custom attributes like an object, but in mongo they will have the name defined in taiga. - "custom_attributes": custom_attributes, + # We can get all the custom attributes like an object, but in mongo they will have the name defined in taiga. + "custom_attributes": custom_attributes, } doc.update(milestone_data) # Return the parsed task data as a dictionary return doc - - - - - -#Most fields dont appear when creating the user story from zero, they appear once we link it to an epic -def parse_taiga_userstory_event(raw_payload: Dict, prj: str) -> Dict: - ''' +# Most fields dont appear when creating the user story from zero, they appear once we link it to an epic +def parse_taiga_userstory_event(raw_payload: Dict, prj: str) -> Dict: + """ Function to parse a taiga userstory event payload. - ''' + """ # Extract the relevant fields from the raw payload - project_id= raw_payload.get("data",{}).get("project",{}).get("id", "") - userstory_id= raw_payload.get("data",{}).get("id", "") - team_name = raw_payload.get("data",{}).get("project", {}).get("name", "") - event_type= raw_payload.get("type","") - action_type= raw_payload.get("action","") - subject = raw_payload.get("data",{}).get("subject", "") + project_id = raw_payload.get("data", {}).get("project", {}).get("id", "") + userstory_id = raw_payload.get("data", {}).get("id", "") + team_name = raw_payload.get("data", {}).get("project", {}).get("name", "") + event_type = raw_payload.get("type", "") + action_type = raw_payload.get("action", "") + subject = raw_payload.get("data", {}).get("subject", "") status = raw_payload.get("data", {}).get("status", {}).get("name", "") is_closed = raw_payload.get("is_closed", False) - modified_date = to_madrid_local(raw_payload.get("data", {}).get("modified_date", "")) + modified_date = to_madrid_local( + raw_payload.get("data", {}).get("modified_date", "") + ) created_date = to_madrid_local(raw_payload.get("data", {}).get("created_date", "")) assigned_by = raw_payload.get("by", {}).get("username", "") # Extract all the custom attributes from the payload, if they exist - custom_attributes = raw_payload.get("data", {}).get("custom_attributes_values", {}) + custom_attributes = raw_payload.get("data", {}).get("custom_attributes_values", {}) if custom_attributes is None: custom_attributes = {} - - description= raw_payload.get("data", {}).get("description", "") - #If the pattern "AS - A - I WANT - SO THAT" is used in the description, the vañue of pattern will be True, if not, it will be False + + description = raw_payload.get("data", {}).get("description", "") + # If the pattern "AS - A - I WANT - SO THAT" is used in the description, the vañue of pattern will be True, if not, it will be False pattern = r"as\s+(.*?)\s+i want\s+(.*?)\s+so that\s+(.*)" match = re.search(pattern, description, re.IGNORECASE) if match: pattern_in_description = True else: pattern_in_description = False - + # If the userstory has a milestone associated while created, we will get the values, if not, we will set them to None - if raw_payload.get("data",{}).get("milestone",{}) != None: - - milestone_id= raw_payload.get("data",{}).get("milestone",{}).get("id", "") - milestone_name= raw_payload.get("data",{}).get("milestone",{}).get("name", "") - milestone_closed= raw_payload.get("data",{}).get("milestone",{}).get("closed", "") - milestone_created_date= to_madrid_local(raw_payload.get("data",{}).get("milestone",{}).get("created_date", "")) - milestone_modified_date= to_madrid_local(raw_payload.get("data",{}).get("milestone",{}).get("modified_date", "")) - milestone_modified_date = to_madrid_local(milestone_modified_date) if milestone_modified_date else "" - estimated_start= to_madrid_local(raw_payload.get("data",{}).get("milestone",{}).get("estimated_start", "")) - estimated_finish= to_madrid_local(raw_payload.get("data",{}).get("milestone",{}).get("estimated_finish", "")) - - milestone_data= milestone_stats(project_id, milestone_id, prj) - - - + if raw_payload.get("data", {}).get("milestone", {}) is not None: + + milestone_id = raw_payload.get("data", {}).get("milestone", {}).get("id", "") + milestone_name = ( + raw_payload.get("data", {}).get("milestone", {}).get("name", "") + ) + milestone_closed = ( + raw_payload.get("data", {}).get("milestone", {}).get("closed", "") + ) + milestone_created_date = to_madrid_local( + raw_payload.get("data", {}).get("milestone", {}).get("created_date", "") + ) + milestone_modified_date = to_madrid_local( + raw_payload.get("data", {}).get("milestone", {}).get("modified_date", "") + ) + milestone_modified_date = ( + to_madrid_local(milestone_modified_date) if milestone_modified_date else "" + ) + estimated_start = to_madrid_local( + raw_payload.get("data", {}).get("milestone", {}).get("estimated_start", "") + ) + estimated_finish = to_madrid_local( + raw_payload.get("data", {}).get("milestone", {}).get("estimated_finish", "") + ) + + milestone_data = milestone_stats(project_id, milestone_id, prj) + else: - milestone_id= "" - milestone_name= "" - milestone_closed= "" - milestone_created_date= "" - milestone_modified_date= "" - estimated_start= "" - estimated_finish= "" + milestone_id = "" + milestone_name = "" + milestone_closed = "" + milestone_created_date = "" + milestone_modified_date = "" + estimated_start = "" + estimated_finish = "" milestone_data = {} - - priority = raw_payload.get("data", {}).get("custom_attributes_values", {}).get("Priority", "") - points_list = raw_payload.get("data",{}).get("points", []) + priority = custom_attributes.get("Priority", "") + points_list = raw_payload.get("data", {}).get("points", []) sum_points = sum(p.get("value") or 0 for p in points_list) - # Create a dictionary with all the attributes of the user story + # Create a dictionary with all the attributes of the user story doc = { "project_id": project_id, "team_name": team_name, @@ -280,7 +306,7 @@ def parse_taiga_userstory_event(raw_payload: Dict, prj: str) -> Dict: "is_closed": is_closed, "status": status, "created_date": created_date, - "modified_date": modified_date, + "modified_date": modified_date, "total_points": sum_points, "assigned_by": assigned_by, "milestone_id": milestone_id, @@ -290,43 +316,45 @@ def parse_taiga_userstory_event(raw_payload: Dict, prj: str) -> Dict: "milestone_modified_date": milestone_modified_date, "estimated_start": estimated_start, "estimated_finish": estimated_finish, - #We can get all the custom attributes like an object, but in mongo they will have the name defined in taiga. - "custom_attributes": custom_attributes, - #"acceptance_criteria": acceptance_criteria, #TRUE IF THE USER STORY HAS ACCEPTANCE CRITERIA - "pattern": pattern_in_description, - "priority": priority + # We can get all the custom attributes like an object, but in mongo they will have the name defined in taiga. + "custom_attributes": custom_attributes, + # "acceptance_criteria": acceptance_criteria, #TRUE IF THE USER STORY HAS ACCEPTANCE CRITERIA + "pattern": pattern_in_description, + "priority": priority, } - + doc.update(milestone_data) # Return the parsed user story data as a dictionary return doc - - -def parse_taiga_related_userstory_event(raw_payload: Dict, prj: str) -> Dict: - ''' +def parse_taiga_related_userstory_event(raw_payload: Dict, prj: str) -> Dict: + """ Function to parse a taiga related userstory event payload. This related userstory event is triggered when a user story is linked to an epic. - ''' + """ # Extract the relevant fields from the raw payload - userstory_id= raw_payload.get("data",{}).get("user_story",{}).get("id", "") - team_name = raw_payload.get("data",{}).get("epic",{}).get("project", {}).get("name", "") - event_type= raw_payload.get("type","") - finished_date = to_madrid_local(raw_payload.get("data",{}).get("finished_date", "")) - assigned_to = raw_payload.get("data",{}).get("assigned_to",{}).get("username", "") - epic_id= raw_payload.get("data",{}).get("epic",{}).get("id", "") - epic_name= raw_payload.get("data",{}).get("epic",{}).get("subject","") - reference= raw_payload.get("data",{}).get("epic",{}).get("ref", "") + userstory_id = raw_payload.get("data", {}).get("user_story", {}).get("id", "") + team_name = ( + raw_payload.get("data", {}).get("epic", {}).get("project", {}).get("name", "") + ) + event_type = raw_payload.get("type", "") + finished_date = to_madrid_local( + raw_payload.get("data", {}).get("finished_date", "") + ) + assigned_to = raw_payload.get("data", {}).get("assigned_to", {}).get("username", "") + epic_id = raw_payload.get("data", {}).get("epic", {}).get("id", "") + epic_name = raw_payload.get("data", {}).get("epic", {}).get("subject", "") + reference = raw_payload.get("data", {}).get("epic", {}).get("ref", "") assigned_by = raw_payload.get("by", {}).get("username", "") # Create a dictionary with all the attributes of the user story doc = { - "id": userstory_id, + "id": userstory_id, "team_name": team_name, "event_type": event_type, "epic_id": epic_id, - "epic_name": epic_name, + "epic_name": epic_name, "reference": reference, "finished_date": finished_date, "assigned_to": assigned_to, @@ -334,4 +362,3 @@ def parse_taiga_related_userstory_event(raw_payload: Dict, prj: str) -> Dict: } # Return the parsed user story data as a dictionary return doc - diff --git a/deprecated/requirements.old.txt b/deprecated/requirements.old.txt new file mode 100644 index 0000000..e38aac5 Binary files /dev/null and b/deprecated/requirements.old.txt differ diff --git a/deprecated/test.py b/deprecated/test.py index 9c0b131..67d9348 100644 --- a/deprecated/test.py +++ b/deprecated/test.py @@ -30,14 +30,12 @@ API_BASE = "https://api.taiga.io/api/v1" - - - def get_by_slug(slug: str, token: str) -> Dict: """Retorna el JSON del projecte pel seu slug (una sola petició).""" h = {"Authorization": f"Bearer {token}"} - r = requests.get(f"{API_BASE}/projects/by_slug", headers=h, - params={"slug": slug}, timeout=10) + r = requests.get( + f"{API_BASE}/projects/by_slug", headers=h, params={"slug": slug}, timeout=10 + ) if r.status_code == 404: sys.exit(f"Slug «{slug}» no existeix o no és visible per aquest usuari.") r.raise_for_status() @@ -48,10 +46,11 @@ def list_user_projects(token: str) -> List[Dict]: """Llista tots els projectes on l’usuari autenticat és membre.""" h = { "Authorization": f"Bearer {token}", - "x-disable-pagination": "True", # rebem tot en una sola resposta + "x-disable-pagination": "True", # rebem tot en una sola resposta } - r = requests.get(f"{API_BASE}/projects", headers=h, - params={"member": "me"}, timeout=10) + r = requests.get( + f"{API_BASE}/projects", headers=h, params={"member": "me"}, timeout=10 + ) r.raise_for_status() return r.json() @@ -75,12 +74,20 @@ def main(argv: List[str] | None = None) -> None: description="Fetch Taiga project id by slug or by display name." ) ap.add_argument("--slug", help="Slug exacte del projecte (mètode preferit).") - ap.add_argument("--name", - help="Nom visible del projecte (case-insensitive). S'usa si --slug no és present.") - ap.add_argument("--user", default=os.getenv("TAIGA_USERNAME"), - help="Usuari Taiga (o variable d’entorn TAIGA_USERNAME).") - ap.add_argument("--password", default=os.getenv("TAIGA_PASSWORD"), - help="Contrasenya Taiga (o variable d’entorn TAIGA_PASSWORD).") + ap.add_argument( + "--name", + help="Nom visible del projecte (case-insensitive). S'usa si --slug no és present.", + ) + ap.add_argument( + "--user", + default=os.getenv("TAIGA_USERNAME"), + help="Usuari Taiga (o variable d’entorn TAIGA_USERNAME).", + ) + ap.add_argument( + "--password", + default=os.getenv("TAIGA_PASSWORD"), + help="Contrasenya Taiga (o variable d’entorn TAIGA_PASSWORD).", + ) args = ap.parse_args(argv) # Validacions bàsiques @@ -88,8 +95,9 @@ def main(argv: List[str] | None = None) -> None: ap.error("Cal indicar --slug o --name") if not (args.user and args.password): - ap.error("Credencials requerides: --user/--password o variables env TAIGA_USERNAME/TAIGA_PASSWORD") - + ap.error( + "Credencials requerides: --user/--password o variables env TAIGA_USERNAME/TAIGA_PASSWORD" + ) # 2. Recuperació del projecte if args.slug: @@ -103,4 +111,4 @@ def main(argv: List[str] | None = None) -> None: # 3. Resultat final print("\n===== RESULTAT =====") - print(f"Nom del projecte : {p + print(f"Nom del projecte : {project['name']}") diff --git a/docker-compose.yml b/docker-compose.yml index 143bb4f..11bbf69 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,23 +1,23 @@ services: - LD_connect: + ld_connect: build: - context: . - dockerfile: Dockerfile + context: . + dockerfile: dockerfile container_name: LDConnect environment: - EVAL_HOST=ld_eval - EVAL_PORT=5001 env_file: - - .env + - .env networks: - - qrapids + - qrapids depends_on: - mongodb: + mongodb: condition: service_started ports: - - "127.0.0.1:5000:5000" + - "127.0.0.1:5000:5000" networks: qrapids: - external: true + external: true diff --git a/pyproject.toml b/pyproject.toml index cc44f6f..5fad333 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,3 +19,6 @@ omit = [ [tool.coverage.report] show_missing = true fail_under = 80 + +[tool.ruff] +extend-exclude = ["deprecated"] diff --git a/requirements.txt b/requirements.txt index b369a4c..8c0dc96 100644 Binary files a/requirements.txt and b/requirements.txt differ diff --git a/routes/API_publisher/API_event_publisher.py b/routes/API_publisher/API_event_publisher.py index 5efb120..2969921 100644 --- a/routes/API_publisher/API_event_publisher.py +++ b/routes/API_publisher/API_event_publisher.py @@ -5,15 +5,17 @@ logger = logging.getLogger(__name__) -def notify_eval_push(event_type: str,prj: str ,author_login: str , quality_model: str)-> None: - ''' +def notify_eval_push( + event_type: str, prj: str, author_login: str, quality_model: str +) -> None: + """ Function used to notify Component LD_Eval about the event that has been pushed to the database. - ''' - + """ + host = os.getenv("EVAL_HOST", "localhost") port = os.getenv("EVAL_PORT", "5001") - url = f"http://{host}:{port}/api/event" - + url = f"http://{host}:{port}/api/event" + event_data = { "event_type": event_type, "prj": prj, @@ -26,4 +28,4 @@ def notify_eval_push(event_type: str,prj: str ,author_login: str , quality_model resp.raise_for_status() logger.info("LD_Eval responded with %s: %s", resp.status_code, resp.json()) except requests.RequestException as e: - logger.error("Error notifying LD_Eval at %s: %s", url, e) \ No newline at end of file + logger.error("Error notifying LD_Eval at %s: %s", url, e) diff --git a/routes/excel_routes.py b/routes/excel_routes.py index 7cbbc4c..ca10c61 100644 --- a/routes/excel_routes.py +++ b/routes/excel_routes.py @@ -1,7 +1,6 @@ -from flask import Blueprint, request, jsonify +from flask import Blueprint, make_response, request, jsonify from datasources.excel_handler import parse_excel_event from database.mongo_client import get_collection -from routes.API_publisher.API_event_publisher import notify_eval_push from config.logger_config import setup_logging import logging @@ -9,41 +8,42 @@ logger = logging.getLogger(__name__) - excel_bp = Blueprint("excel_bp", __name__) @excel_bp.route("/webhook/excel", methods=["POST"]) def excel_webhook(): - + logger.info("Received Excel webhook request.") - + # Get the raw JSON payload from the request raw_json = request.get_json() if not raw_json: logger.warning("Excel webhook called without JSON payload.") - return {"error": "No JSON received"}, 400 - + return make_response(jsonify({"error": "No JSON received"}), 400) + # Read the query parameters from the request - prj = request.args.get("prj", type=str) - quality_model = request.args.get("quality_model", type=str) # otional, if not provided, we have to use the default one - + prj = request.args.get("prj", type=str) + quality_model = request.args.get( + "quality_model", type=str + ) # otional, if not provided, we have to use the default one + if not prj: logger.warning("Missing required query param: prj") return jsonify({"error": "prj is required as query parameter"}), 400 - # Parse the raw JSON payload using the parse_excel_event function - parsed_data = parse_excel_event(raw_json, prj, quality_model) + parsed_data = parse_excel_event(raw_json, prj, quality_model) if "error" in parsed_data: - return parsed_data, 400 + return make_response(jsonify(parsed_data), 400) logger.info("Excel webhook request processed successfully.") - # Create the collection name based on the project ID + # Create the collection name based on the project ID collection_name = f"{prj}_sheets" event_name = "sheets_activity" - author_login = '' #username of the author - + # author_login = "" # username of the author + logger.info(f"Processing Excel event: {event_name} for team: {prj} with quality_model: {quality_model}") + coll = get_collection(collection_name) # #COMMUNICATION WITH LD_EVAL USING API @@ -53,13 +53,9 @@ def excel_webhook(): # except Exception as e: # logger.error(f"Error notifying LD_EVAL: {e}") # return {"status": "error", "message": str(e)}, 500 - - + logger.info(f"Inserting Excel activity document for team {prj}") # Insert the parsed data into the MongoDB collection coll.insert_one(parsed_data) - - return jsonify({"status": "OK"}) - - + return jsonify({"status": "OK"}) diff --git a/routes/github_routes.py b/routes/github_routes.py index 899437a..9cae38a 100644 --- a/routes/github_routes.py +++ b/routes/github_routes.py @@ -1,4 +1,4 @@ -from flask import Blueprint, request, jsonify +from flask import Blueprint, make_response, request, jsonify from datasources.github_handler import parse_github_event from database.mongo_client import get_collection from config.settings import GITHUB_SIGNATURE_KEY @@ -12,50 +12,62 @@ github_bp = Blueprint("github_bp", __name__) + @github_bp.route("/webhook/github", methods=["POST"]) def github_webhook(): - + logger.info("Received Github webhook request.") - - # Signature verfication, in the definition of the webhook we must have the same value as in the .env file - secret=GITHUB_SIGNATURE_KEY.encode() + + # Signature verfication, in the definition of the webhook we must have the same value as in the .env file + secret = GITHUB_SIGNATURE_KEY.encode() if not verify_github_signature(request, secret): logger.warning("Invalid Github webhook signature.") - return jsonify({"error": "Invalid Signature"}), 403 - + return jsonify({"error": "Invalid Signature"}), 403 + # Get the raw JSON payload from the request raw_payload = request.get_json() if not raw_payload: logger.warning("Github webhook called without JSON payload.") return {"error": "No JSON received"}, 400 - - + # Read the query parameters from the request - prj = request.args.get("prj", type=str) - quality_model = request.args.get("quality_model", type=str) # otional, if not provided, we have to use the default one + prj = request.args.get("prj", type=str) + quality_model = request.args.get( + "quality_model", type=str + ) # otional, if not provided, we have to use the default one if not prj: logger.warning("Missing required query param: prj") return jsonify({"error": "prj is required as query parameter"}), 400 - - + # We read the event name from the GitHub header, not in the JSON event_name = request.headers.get("X-GitHub-Event") - raw_payload["X-GitHub-Event"] = event_name # Put it in the JSON so parse function sees it + raw_payload["X-GitHub-Event"] = ( + event_name # Put it in the JSON so parse function sees it + ) # Parse the raw JSON payload using the parse_github_event function parsed_data = parse_github_event(raw_payload, prj) - logger.info(f"Github webhook request processed successfully for team {prj}.") - + logger.info("Github webhook request processed successfully for team %s.", prj) + if parsed_data.get("ignored"): - return {"status": "ignored", "event": parsed_data["event"]}, 200 + return make_response(jsonify({"status": "ignored", "event": parsed_data["event"]}), 200) if "error" in parsed_data: - return parsed_data, 400 + return make_response(jsonify(parsed_data), 400) + team_name = parsed_data[ + "team_name" + ] # We wont use this, we will use the external_id instead as its a CENTRALIZED ID + logger.info( + "Processing Github event: %s for team: %s (external_id: %s)", + parsed_data["event"], + team_name, + prj, + ) + event_label = parsed_data["event"] # This is either "commit" or "issue" + author_login = parsed_data["sender_info"][ + "login" + ] # username of the author of the commit or issue - team_name = parsed_data["team_name"] #We wont use this, we will use the external_id instead as its a CENTRALIZED ID - event_label = parsed_data["event"] #This is either "commit" or "issue" - author_login = parsed_data["sender_info"]["login"] #username of the author of the commit or issue - # Decide the name of the MongoDB collection to write to, depending on the event type if event_label == "commit": collection_name = f"github_{prj}.commits" @@ -63,17 +75,21 @@ def github_webhook(): collection_name = f"{prj}_issues" elif event_label == "pull_request": collection_name = f"github_{prj}.pull_requests" - + coll = get_collection(collection_name) # # COMMUNICATION WITH LD_EVAL USING API - logger.info(f"Notifying LD_EVAL about event: {event_name} for team with external_id: {prj} with quality_model: {quality_model}") + logger.info( + "Notifying LD_EVAL about event: %s for team with external_id: %s with quality_model: %s", + event_name, + prj, + quality_model, + ) try: notify_eval_push(event_name, prj, author_login, quality_model) except Exception as e: - logger.error(f"Error notifying LD_EVAL: {e}") + logger.error("Error notifying LD_EVAL: %s", e) return {"status": "error", "message": str(e)}, 500 - # If it's a commit push, we may have multiple commits. We need to insert each one separately. if "commits" in parsed_data: @@ -85,26 +101,27 @@ def github_webhook(): commit_doc["event"] = parsed_data["event"] commit_doc["repo_name"] = parsed_data["repo_name"] - logger.debug(f"Inserting commit document: {commit_doc}") + logger.debug("Inserting GitHub commit document for team %s", prj) coll.insert_one(commit_doc) - logger.info(f"Inserting in MongoDB Github commit for team {prj}") + logger.info("Inserting in MongoDB Github commit for team %s", prj) - return {"status": "ok", "message": "Commits inserted"}, 200 + return make_response(jsonify({"status": "ok", "message": "Commits inserted"}), 200) # If it's an issue event, we insert the issue document elif "issue" in parsed_data: parsed_data["prj"] = prj coll.insert_one(parsed_data) - logger.info(f"Inserting in MongoDB Github issue for team {prj}") - return {"status": "ok", "message": "Issue inserted"}, 200 - - + logger.info("Inserting in MongoDB Github issue for team %s", prj) + return make_response(jsonify({"status": "ok", "message": "Issue inserted"}), 200) + elif "pull_request" in parsed_data: parsed_data["prj"] = prj coll.insert_one(parsed_data) - logger.info(f"Inserting in MongoDB Github closed pull request for team {prj}") + logger.info( + "Inserting in MongoDB Github closed pull request for team %s", prj + ) return {"status": "ok", "message": "Pull request inserted"}, 200 - #If its neither a commit or a issue + # If its neither a commit or a issue coll.insert_one(parsed_data) return {"status": "ok", "message": "Stored event doc"}, 200 diff --git a/routes/taiga_routes.py b/routes/taiga_routes.py index e57c5db..6e78495 100644 --- a/routes/taiga_routes.py +++ b/routes/taiga_routes.py @@ -6,40 +6,44 @@ from routes.verify_signature.verify_signature_taiga import verify_taiga_signature import logging -logger = logging.getLogger(__name__) +logger = logging.getLogger(__name__) taiga_bp = Blueprint("taiga_bp", __name__) @taiga_bp.route("/webhook/taiga", methods=["POST"]) def taiga_webhook(): - + logger.info("Received Taiga webhook request.") - - # Signature verfication, in the definition of the webhook we must have the same value as in the .env file - secret=TAIGA_SIGNATURE_KEY.encode() + + # Signature verfication, in the definition of the webhook we must have the same value as in the .env file + secret = TAIGA_SIGNATURE_KEY.encode() if not verify_taiga_signature(request, secret): logger.warning("Invalid Taiga webhook signature.") - return jsonify({"error": "Invalid Signature"}), 403 - + return jsonify({"error": "Invalid Signature"}), 403 + # Get the raw JSON payload from the request raw_payload = request.json if not raw_payload: logger.warning("Taiga webhook called without JSON payload.") return jsonify({"error": "No JSON"}), 400 - # Read the query parameters from the request prj = request.args.get("prj", type=str) - quality_model = request.args.get("quality_model", type=str) # otional, if not provided, we have to use the default one - + quality_model = request.args.get( + "quality_model", type=str + ) # otional, if not provided, we have to use the default one # Get important values from the payload - event_type= raw_payload.get("type","") - action_type= raw_payload.get("action","") - id = raw_payload.get("data",{}).get("id", "") - team_name = raw_payload.get("data",{}).get("project", {}).get("name", "") - + event_type = raw_payload.get("type", "") + action_type = raw_payload.get("action", "") + id = raw_payload.get("data", {}).get("id", "") + logger.info( + "Processing Taiga event type=%s action=%s project=%s", + event_type, + action_type, + prj, + ) # Decide the Mongo collection name to write to, depending on the event type if event_type in ["userstory", "relateduserstory"]: @@ -55,83 +59,78 @@ def taiga_webhook(): coll = get_collection(collection_name) - #Handle the deletion of a document before we parse the payload, to avoid data errors + # Handle the deletion of a document before we parse the payload, to avoid data errors if action_type == "delete": - logger.info(f"Deleting document from {collection_name}. ID={id}") + logger.info("Deleting document from %s. ID=%s", collection_name, id) if not id: return jsonify({"error": "No object ID"}), 400 coll.delete_one({f"{event_type}_id": id}) - logger.info(f"Document with {event_type}={id} has been deleted.") + logger.info("Document with %s=%s has been deleted.", event_type, id) return jsonify({"status": "ok"}), 200 - - - # Parse the raw JSON payload using the parse_taiga_event function + # Parse the raw JSON payload using the parse_taiga_event function parsed_data = parse_taiga_event(raw_payload, prj) logger.info("Taiga webhook request processed successfully.") - author_login = parsed_data["assigned_by"] #username of the author of the commit or issue + author_login = parsed_data[ + "assigned_by" + ] # username of the author of the commit or issue - #If the event is a user story, identify the user story ID and upsert/insert it in the collection + # If the event is a user story, identify the user story ID and upsert/insert it in the collection if event_type in ["userstory", "relateduserstory"]: # UP-SERT user stories in the same collection user_story_id = parsed_data.get("userstory_id") if not user_story_id: return jsonify({"error": "No user story ID"}), 400 - logger.info(f"Upserting user story with ID: {user_story_id}") + logger.info("Upserting user story with ID: %s", user_story_id) parsed_data["prj"] = prj result = coll.update_one( - {"userstory_id": user_story_id}, - {"$set": parsed_data}, - upsert=True + {"userstory_id": user_story_id}, {"$set": parsed_data}, upsert=True ) - logger.info(f"Inserting in MongoDB Taiga userstory for team {prj}") - - - #If the event is a taks , identify the task ID and upsert/insert it in the collection + logger.info("Inserting in MongoDB Taiga userstory for team %s", prj) + + # If the event is a taks , identify the task ID and upsert/insert it in the collection elif event_type == "task": - - #if in the parsed data is_closed is true, means the task is closed, so we have to update the points of the user story - #if parsed_data.get("is_closed") == True and parsed_data.get("userstory_is_closed") == True: - - - - + + # if in the parsed data is_closed is true, means the task is closed, so we have to update the points of the user story + # if parsed_data.get("is_closed") == True and parsed_data.get("userstory_is_closed") == True: + coll = get_collection(collection_name) task_id = parsed_data.get("task_id") if not task_id: return jsonify({"error": "No task ID"}), 400 - logger.info(f"Upserting task with ID: {task_id}") + logger.info("Upserting task with ID: %s", task_id) # Upsert instead of insert - parsed_data["prj"] = prj - result = coll.update_one( - {"task_id": task_id}, - {"$set": parsed_data}, - upsert=True - ) - logger.info(f"Inserting in MongoDB Taiga task for team {prj}") - - - #If the event is an epic, identify the epic ID and upsert/insert it in the collection + result = parsed_data["prj"] = prj + logger.info( + "Upserting task with ID: %s for team %s in collection %s result: %s", + task_id, + prj, + collection_name, + result, + ) + coll.update_one( + {"task_id": task_id}, {"$set": parsed_data}, upsert=True + ) + logger.info("Inserting in MongoDB Taiga task for team %s", prj) + + # If the event is an epic, identify the epic ID and upsert/insert it in the collection elif event_type == "epic": coll = get_collection(collection_name) epic_id = parsed_data.get("epic_id") if not epic_id: return jsonify({"error": "No epic ID"}), 400 - logger.info(f"Upserting epic with ID: {epic_id}") + logger.info("Upserting epic with ID: %s", epic_id) # Upsert instead of insert parsed_data["prj"] = prj - result = coll.update_one( - {"epic_id": epic_id}, - {"$set": parsed_data}, - upsert=True - ) - logger.info(f"Inserting in MongoDB Taiga epic for team {prj}") - - + coll.update_one( + {"epic_id": epic_id}, {"$set": parsed_data}, upsert=True + ) + logger.info("Inserting in MongoDB Taiga epic for team %s", prj) + # If the event is an issue, identify the issue ID and upsert/insert it in the collection elif event_type == "issue": coll = get_collection(collection_name) @@ -139,30 +138,30 @@ def taiga_webhook(): if not issue_id: return jsonify({"error": "No issue ID"}), 400 - logger.info(f"Upserting issue with ID: {issue_id}") + logger.info("Upserting issue with ID: %s", issue_id) parsed_data["prj"] = prj # Upsert instead of insert - result = coll.update_one( - {"issue_id": issue_id}, - {"$set": parsed_data}, - upsert=True - ) - logger.info(f"Inserting in MongoDB Taiga issue for team {prj}") - - + coll.update_one( + {"issue_id": issue_id}, {"$set": parsed_data}, upsert=True + ) + logger.info("Inserting in MongoDB Taiga issue for team %s", prj) + else: # If the event is not one of the above, we will insert it as a new document parsed_data["prj"] = prj - inserted_id = coll.insert_one(parsed_data).inserted_id - - - #COMMUNICATION WITH LD_EVAL USING API - logger.info(f"Notifying LD_EVAL about event: {event_type} for team with external_id: {prj} with quality_model: {quality_model}") + coll.insert_one(parsed_data) + + # COMMUNICATION WITH LD_EVAL USING API + logger.info( + "Notifying LD_EVAL about event: %s for team with external_id: %s with quality_model: %s", + event_type, + prj, + quality_model, + ) try: notify_eval_push(event_type, prj, author_login, quality_model) except Exception as e: - logger.error(f"Error notifying LD_EVAL: {e}") + logger.error("Error notifying LD_EVAL: %s", e) return jsonify({"error": "Failed to notify LD_EVAL"}), 500 - - + return jsonify({"status": "ok"}), 200 diff --git a/routes/verify_signature/verify_signature_github.py b/routes/verify_signature/verify_signature_github.py index c170eee..9bc543a 100644 --- a/routes/verify_signature/verify_signature_github.py +++ b/routes/verify_signature/verify_signature_github.py @@ -5,20 +5,18 @@ def verify_github_signature(request, secret): """ Validates the GitHub HMAC signature on the incoming request. - + """ # The signature sent by GitHub, e.g. "sha256=abc123..." signature_header = request.headers.get("X-Hub-Signature-256", "") - + # The raw request body (as bytes), essential for computing HMAC raw_body = request.data - + # Compute our own HMAC sha256 - expected_signature = "sha256=" + hmac.new( - secret, - raw_body, - hashlib.sha256 - ).hexdigest() + expected_signature = ( + "sha256=" + hmac.new(secret, raw_body, hashlib.sha256).hexdigest() + ) # Safely compare the two return hmac.compare_digest(expected_signature, signature_header) diff --git a/routes/verify_signature/verify_signature_taiga.py b/routes/verify_signature/verify_signature_taiga.py index 0ea273e..0a4e9db 100644 --- a/routes/verify_signature/verify_signature_taiga.py +++ b/routes/verify_signature/verify_signature_taiga.py @@ -1,6 +1,7 @@ import hashlib import hmac + def verify_taiga_signature(request, secret): """ Validates the Taiga HMAC-SHA1 signature on the incoming request. diff --git a/template.env b/template.env index 0f4440c..c4fffb5 100644 --- a/template.env +++ b/template.env @@ -1,5 +1,5 @@ # Set to True to enable debug mode, which will print more detailed logs and error messages. Set to False for production use. -DEBUG=False +DEBUG=False # Flask server settings (only used when running python app.py directly) FLASK_DEBUG=false @@ -13,24 +13,16 @@ MONGO_USER= MONGO_PASS= MONGO_AUTHSRC= +CREDENTIALS_FILE=config_files/credentials_config.json -//#The github token should be the token of an administrator of all the repositories/organizations of all the students -//Also when creating this token we need to select 'admin:repo_hook' and also 'repo' to have full access to the commits stats - - +#### GitHub configuration #This is the secret key that we will use to verify the signature. I must be the same as the one in the webhook -GITHUB_SIGNATURE_KEY="myGHS3cr3t!" +GITHUB_SIGNATURE_KEY="myGHS3cr3t!" # GitHub API URL (default: https://api.github.com, change for GitHub Enterprise) GITHUB_API_URL=https://api.github.com - - -#This is the secret key that we will use to verify the signature. I must be the same as the one in the webhook -TAIGA_SIGNATURE_KEY="asdqasfs" - - #### Configuración de Taiga # URL de la API de Taiga @@ -41,6 +33,8 @@ TAIGA_API_URL=https://api.taiga.io/api/v1 # URL de autenticación de Taiga (defaults to TAIGA_API_URL if not set) TAIGA_AUTH_URL=https://api.taiga.io/api/v1 +#This is the secret key that we will use to verify the signature. I must be the same as the one in the webhook +TAIGA_SIGNATURE_KEY="asdqasfs" # URLS de los webhooks WEBHOOK_URL_GITHUB="https://gessi-dashboard.essi.upc.edu/webhook/github" diff --git a/tests/conftest.py b/tests/conftest.py index 01c60a7..6e7d109 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,10 @@ """ Shared pytest fixtures for the LD_Connect_Event test suite. """ -import os, json, pytest + +import os +import json +import pytest # ── Set required env vars BEFORE any application module is imported ────────── os.environ.setdefault("GITHUB_SIGNATURE_KEY", "test-github-secret") @@ -16,6 +19,7 @@ # ── Fixtures ───────────────────────────────────────────────────────────────── + @pytest.fixture def sample_credentials_config(tmp_path): """Create a temporary credentials JSON file and return its path.""" @@ -24,14 +28,14 @@ def sample_credentials_config(tmp_path): "github_token": "ghp_FAKETOKEN123", "taiga_user": "tuser", "taiga_password": "tpass", - "teams": ["TeamAlpha", "TeamBeta"] + "teams": ["TeamAlpha", "TeamBeta"], }, "course_b": { "github_token": "ghp_FAKETOKEN456", "taiga_user": "", "taiga_password": "", - "teams": ["TeamGamma"] - } + "teams": ["TeamGamma"], + }, } p = tmp_path / "creds.json" p.write_text(json.dumps(data)) @@ -42,6 +46,7 @@ def sample_credentials_config(tmp_path): def flask_app(): """Create a Flask test application.""" from app import create_app + app = create_app() app.config["TESTING"] = True return app @@ -55,6 +60,7 @@ def client(flask_app): # ── Sample payloads ────────────────────────────────────────────────────────── + @pytest.fixture def github_push_payload(): """Minimal GitHub push webhook payload.""" @@ -67,7 +73,7 @@ def github_push_payload(): "login": "devuser", "url": "https://api.github.com/users/devuser", "type": "User", - "site_admin": False + "site_admin": False, }, "commits": [ { @@ -78,10 +84,10 @@ def github_push_payload(): "author": { "username": "devuser", "name": "Dev User", - "email": "dev@example.com" - } + "email": "dev@example.com", + }, } - ] + ], } @@ -98,15 +104,15 @@ def github_issue_payload(): "login": "issueuser", "url": "https://api.github.com/users/issueuser", "type": "User", - "site_admin": False + "site_admin": False, }, "issue": { "number": 10, "title": "Bug in login", "state": "open", "body": "Login fails with error 500", - "user": {"login": "issueuser", "id": 2} - } + "user": {"login": "issueuser", "id": 2}, + }, } @@ -123,7 +129,7 @@ def github_pr_payload(): "login": "pruser", "url": "https://api.github.com/users/pruser", "type": "User", - "site_admin": False + "site_admin": False, }, "pull_request": { "number": 5, @@ -133,8 +139,8 @@ def github_pr_payload(): "merged": True, "merged_by": {"login": "merger"}, "assignee": {"login": "pruser"}, - "requested_reviewers": [{"login": "reviewer1"}] - } + "requested_reviewers": [{"login": "reviewer1"}], + }, } @@ -162,11 +168,11 @@ def taiga_task_payload(): "created_date": "2025-05-01T00:00:00Z", "modified_date": "2025-06-01T00:00:00Z", "estimated_start": "2025-05-01T00:00:00Z", - "estimated_finish": "2025-06-01T00:00:00Z" + "estimated_finish": "2025-06-01T00:00:00Z", }, "assigned_to": {"username": "dev1"}, - "custom_attributes_values": {"story_points": 5} - } + "custom_attributes_values": {"story_points": 5}, + }, } @@ -191,9 +197,9 @@ def taiga_issue_payload(): "modified_date": "2025-06-10T10:00:00Z", "created_date": "2025-06-01T08:00:00Z", "finished_date": None, - "assigned_to": {"username": "dev2"} + "assigned_to": {"username": "dev2"}, }, - "is_closed": False + "is_closed": False, } @@ -211,9 +217,9 @@ def taiga_epic_payload(): "status": {"name": "New"}, "is_closed": False, "modified_date": "2025-06-10T10:00:00Z", - "created_date": "2025-06-01T08:00:00Z" + "created_date": "2025-06-01T08:00:00Z", }, - "is_closed": False + "is_closed": False, } @@ -242,10 +248,10 @@ def taiga_userstory_payload(): "created_date": "2025-05-01T00:00:00Z", "modified_date": "2025-06-01T00:00:00Z", "estimated_start": "2025-05-01T00:00:00Z", - "estimated_finish": "2025-06-01T00:00:00Z" - } + "estimated_finish": "2025-06-01T00:00:00Z", + }, }, - "is_closed": False + "is_closed": False, } @@ -262,11 +268,11 @@ def taiga_related_userstory_payload(): "id": 300, "subject": "Epic feature", "ref": 1, - "project": {"name": "TestProject"} + "project": {"name": "TestProject"}, }, "finished_date": "2025-07-01T12:00:00Z", - "assigned_to": {"username": "dev1"} - } + "assigned_to": {"username": "dev1"}, + }, } @@ -283,5 +289,5 @@ def excel_payload(): "epic": "Epic 1", "members": ["Alice", "Bob", ""], "memberHours": [3, 2], - "configRange": [0, 0, 0, 0, 5, 0, 0, 0] + "configRange": [0, 0, 0, 0, 5, 0, 0, 0], } diff --git a/tests/test_api_event_publisher.py b/tests/test_api_event_publisher.py index 2270c2f..f06f4a7 100644 --- a/tests/test_api_event_publisher.py +++ b/tests/test_api_event_publisher.py @@ -1,5 +1,5 @@ """Tests for routes/API_publisher/API_event_publisher.py""" -import pytest + from unittest.mock import patch, MagicMock @@ -27,6 +27,7 @@ def test_successful_notification(self, mock_post): def test_network_error_does_not_raise(self, mock_post): import requests as req from routes.API_publisher.API_event_publisher import notify_eval_push + mock_post.side_effect = req.RequestException("Connection refused") # Should not raise — it logs and continues notify_eval_push("push", "P", "u", "qm") diff --git a/tests/test_app.py b/tests/test_app.py index 3394afc..b01b7e3 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -1,5 +1,4 @@ """Tests for app.py""" -import pytest class TestCreateApp: diff --git a/tests/test_credentials_loader.py b/tests/test_credentials_loader.py index 23173c8..a24ccd4 100644 --- a/tests/test_credentials_loader.py +++ b/tests/test_credentials_loader.py @@ -1,5 +1,7 @@ """Tests for config/credentials_loader.py""" -import json, os, pytest + +import json +import pytest from unittest.mock import patch @@ -7,6 +9,7 @@ class TestLoad: def test_load_returns_parsed_json(self, sample_credentials_config): with patch("config.credentials_loader.CONFIG_FILE", sample_credentials_config): from config.credentials_loader import load + data = load() assert "course_a" in data assert data["course_a"]["github_token"] == "ghp_FAKETOKEN123" @@ -16,6 +19,7 @@ def test_load_file_not_found(self, tmp_path): bad_path = str(tmp_path / "nonexistent.json") with patch("config.credentials_loader.CONFIG_FILE", bad_path): from config.credentials_loader import load + with pytest.raises(FileNotFoundError): load() @@ -24,6 +28,7 @@ def test_load_invalid_json(self, tmp_path): p.write_text("{invalid json") with patch("config.credentials_loader.CONFIG_FILE", str(p)): from config.credentials_loader import load + with pytest.raises(json.JSONDecodeError): load() @@ -32,29 +37,34 @@ class TestResolve: def test_resolve_existing_project(self, sample_credentials_config): with patch("config.credentials_loader.CONFIG_FILE", sample_credentials_config): from config.credentials_loader import resolve + token = resolve("TeamAlpha", "github_token") assert token == "ghp_FAKETOKEN123" def test_resolve_second_course(self, sample_credentials_config): with patch("config.credentials_loader.CONFIG_FILE", sample_credentials_config): from config.credentials_loader import resolve + token = resolve("TeamGamma", "github_token") assert token == "ghp_FAKETOKEN456" def test_resolve_project_not_found(self, sample_credentials_config): with patch("config.credentials_loader.CONFIG_FILE", sample_credentials_config): from config.credentials_loader import resolve + with pytest.raises(KeyError, match="NonExistentProject"): resolve("NonExistentProject", "github_token") def test_resolve_field_missing_returns_none(self, sample_credentials_config): with patch("config.credentials_loader.CONFIG_FILE", sample_credentials_config): from config.credentials_loader import resolve + result = resolve("TeamAlpha", "nonexistent_field") assert result is None def test_resolve_empty_string_field(self, sample_credentials_config): with patch("config.credentials_loader.CONFIG_FILE", sample_credentials_config): from config.credentials_loader import resolve + result = resolve("TeamGamma", "taiga_user") assert result == "" diff --git a/tests/test_datetime_utils.py b/tests/test_datetime_utils.py index 358a176..75d4f8d 100644 --- a/tests/test_datetime_utils.py +++ b/tests/test_datetime_utils.py @@ -1,5 +1,5 @@ """Tests for utils/datetime_utils.py""" -import pytest + from utils.datetime_utils import to_madrid_local diff --git a/tests/test_delete_webhooks_github.py b/tests/test_delete_webhooks_github.py index 0b4a4db..7a51eae 100644 --- a/tests/test_delete_webhooks_github.py +++ b/tests/test_delete_webhooks_github.py @@ -1,5 +1,5 @@ """Tests for utils/webhook_deletion/delete_webhooks_github.py""" -import pytest + from unittest.mock import patch, MagicMock @@ -35,7 +35,9 @@ class TestDeleteAllGithubWebhooks: @patch("utils.webhook_deletion.delete_webhooks_github.list_github_hooks") @patch("utils.webhook_deletion.delete_webhooks_github.pymongo.MongoClient") def test_deletes_matching_webhooks(self, mock_mongo, mock_list, mock_delete): - from utils.webhook_deletion.delete_webhooks_github import delete_all_github_webhooks + from utils.webhook_deletion.delete_webhooks_github import ( + delete_all_github_webhooks, + ) # Setup mock MongoDB mock_db = MagicMock() @@ -57,7 +59,9 @@ def test_deletes_matching_webhooks(self, mock_mongo, mock_list, mock_delete): @patch("utils.webhook_deletion.delete_webhooks_github.pymongo.MongoClient") def test_http_error_listing_continues(self, mock_mongo, mock_list): import requests - from utils.webhook_deletion.delete_webhooks_github import delete_all_github_webhooks + from utils.webhook_deletion.delete_webhooks_github import ( + delete_all_github_webhooks, + ) mock_db = MagicMock() mock_db.list_collection_names.return_value = ["github_prj.commits"] diff --git a/tests/test_delete_webhooks_taiga.py b/tests/test_delete_webhooks_taiga.py index 2a1cbd7..782a04f 100644 --- a/tests/test_delete_webhooks_taiga.py +++ b/tests/test_delete_webhooks_taiga.py @@ -1,5 +1,5 @@ """Tests for utils/webhook_deletion/delete_webhooks_taiga.py""" -import pytest + from unittest.mock import patch, MagicMock @@ -35,7 +35,9 @@ class TestDeleteAllTaigaWebhooks: @patch("utils.webhook_deletion.delete_webhooks_taiga.list_taiga_hooks") @patch("utils.webhook_deletion.delete_webhooks_taiga.pymongo.MongoClient") def test_deletes_matching_webhooks(self, mock_mongo, mock_list, mock_delete): - from utils.webhook_deletion.delete_webhooks_taiga import delete_all_taiga_webhooks + from utils.webhook_deletion.delete_webhooks_taiga import ( + delete_all_taiga_webhooks, + ) mock_db = MagicMock() mock_db.list_collection_names.return_value = ["taiga_prj.epics", "other"] @@ -55,7 +57,9 @@ def test_deletes_matching_webhooks(self, mock_mongo, mock_list, mock_delete): @patch("utils.webhook_deletion.delete_webhooks_taiga.pymongo.MongoClient") def test_delete_error_continues(self, mock_mongo, mock_list, mock_delete): import requests - from utils.webhook_deletion.delete_webhooks_taiga import delete_all_taiga_webhooks + from utils.webhook_deletion.delete_webhooks_taiga import ( + delete_all_taiga_webhooks, + ) mock_db = MagicMock() mock_db.list_collection_names.return_value = ["taiga_prj.epics"] diff --git a/tests/test_excel_handler.py b/tests/test_excel_handler.py index 1e20da6..00bb04b 100644 --- a/tests/test_excel_handler.py +++ b/tests/test_excel_handler.py @@ -1,5 +1,5 @@ """Tests for datasources/excel_handler.py""" -import pytest + from datasources.excel_handler import parse_excel_event, ACTIVITY_TYPES @@ -44,7 +44,7 @@ def test_empty_members(self): "epic": "", "members": [], "memberHours": [], - "configRange": [] + "configRange": [], } result = parse_excel_event(payload, "TestPrj", "qm1") assert result["members"] == [] @@ -64,7 +64,7 @@ def test_more_members_than_hours(self): "epic": "", "members": ["A", "B"], "memberHours": [1, 2, 3], - "configRange": [] + "configRange": [], } result = parse_excel_event(payload, "P", "qm") assert "hours_A" in result @@ -83,7 +83,7 @@ def test_config_range_shorter_than_activity_types(self): "epic": "", "members": [], "memberHours": [], - "configRange": [10, 20] # Only 2 values for 8 activity types + "configRange": [10, 20], # Only 2 values for 8 activity types } result = parse_excel_event(payload, "P", "qm") assert result["hours_Reunió_d'equip"] == 10 @@ -102,7 +102,7 @@ def test_config_range_with_none_values(self): "epic": "", "members": [], "memberHours": [], - "configRange": [None, 5, None, None, None, None, None, None] + "configRange": [None, 5, None, None, None, None, None, None], } result = parse_excel_event(payload, "P", "qm") assert result["hours_Reunió_d'equip"] == 0 @@ -128,7 +128,7 @@ def test_whitespace_in_members(self): "epic": "", "members": [" Alice ", " ", "Bob"], "memberHours": [1, 2], - "configRange": [] + "configRange": [], } result = parse_excel_event(payload, "P", "qm") assert result["members"] == ["Alice", "Bob"] diff --git a/tests/test_excel_routes.py b/tests/test_excel_routes.py index 8c37dac..3e856ee 100644 --- a/tests/test_excel_routes.py +++ b/tests/test_excel_routes.py @@ -1,12 +1,15 @@ """Tests for routes/excel_routes.py""" -import json, pytest + +import json from unittest.mock import patch, MagicMock class TestExcelWebhook: @patch("routes.excel_routes.get_collection") @patch("routes.excel_routes.parse_excel_event") - def test_successful_excel_webhook(self, mock_parse, mock_coll, client, excel_payload): + def test_successful_excel_webhook( + self, mock_parse, mock_coll, client, excel_payload + ): mock_parse.return_value = {"team": "TestPrj", "activity_type": "Dev"} mock_collection = MagicMock() mock_coll.return_value = mock_collection @@ -14,7 +17,7 @@ def test_successful_excel_webhook(self, mock_parse, mock_coll, client, excel_pay resp = client.post( "/webhook/excel?prj=TestPrj&quality_model=default", data=json.dumps(excel_payload), - content_type="application/json" + content_type="application/json", ) assert resp.status_code == 200 data = resp.get_json() @@ -23,9 +26,7 @@ def test_successful_excel_webhook(self, mock_parse, mock_coll, client, excel_pay def test_missing_json_body(self, client): resp = client.post( - "/webhook/excel?prj=TestPrj", - data="", - content_type="application/json" + "/webhook/excel?prj=TestPrj", data="", content_type="application/json" ) assert resp.status_code == 400 @@ -33,7 +34,7 @@ def test_missing_prj_param(self, client, excel_payload): resp = client.post( "/webhook/excel", data=json.dumps(excel_payload), - content_type="application/json" + content_type="application/json", ) assert resp.status_code == 400 data = resp.get_json() @@ -41,12 +42,14 @@ def test_missing_prj_param(self, client, excel_payload): @patch("routes.excel_routes.get_collection") @patch("routes.excel_routes.parse_excel_event") - def test_parse_error_returns_400(self, mock_parse, mock_coll, client, excel_payload): + def test_parse_error_returns_400( + self, mock_parse, mock_coll, client, excel_payload + ): mock_parse.return_value = {"error": "Invalid data"} resp = client.post( "/webhook/excel?prj=TestPrj", data=json.dumps(excel_payload), - content_type="application/json" + content_type="application/json", ) assert resp.status_code == 400 @@ -59,6 +62,6 @@ def test_collection_name_format(self, mock_parse, mock_coll, client, excel_paylo client.post( "/webhook/excel?prj=MyPrj", data=json.dumps(excel_payload), - content_type="application/json" + content_type="application/json", ) mock_coll.assert_called_with("MyPrj_sheets") diff --git a/tests/test_get_taiga_token.py b/tests/test_get_taiga_token.py index 1d3556d..e971839 100644 --- a/tests/test_get_taiga_token.py +++ b/tests/test_get_taiga_token.py @@ -1,4 +1,5 @@ """Tests for utils/taiga_token/get_taiga_token.py""" + import pytest from unittest.mock import patch, MagicMock diff --git a/tests/test_github_api_call.py b/tests/test_github_api_call.py index 20e159e..6ce58c1 100644 --- a/tests/test_github_api_call.py +++ b/tests/test_github_api_call.py @@ -1,5 +1,5 @@ """Tests for datasources/requests/github_api_call.py""" -import pytest + from unittest.mock import patch, MagicMock @@ -34,7 +34,10 @@ def test_missing_stats_key(self, mock_get, mock_resolve): assert result == {"total": 0, "additions": 0, "deletions": 0} @patch("datasources.requests.github_api_call.resolve", return_value="ghp_TOKEN") - @patch("datasources.requests.github_api_call.requests.get", side_effect=Exception("Network error")) + @patch( + "datasources.requests.github_api_call.requests.get", + side_effect=Exception("Network error"), + ) def test_network_error_returns_zeros(self, mock_get, mock_resolve): from datasources.requests.github_api_call import fetch_commit_stats @@ -44,7 +47,9 @@ def test_network_error_returns_zeros(self, mock_get, mock_resolve): @patch("datasources.requests.github_api_call.resolve", return_value="ghp_TOKEN") @patch("datasources.requests.github_api_call.requests.get") def test_uses_correct_url(self, mock_get, mock_resolve): - from datasources.requests.github_api_call import fetch_commit_stats, GITHUB_API_URL + from datasources.requests.github_api_call import ( + fetch_commit_stats, + ) mock_resp = MagicMock() mock_resp.json.return_value = {"stats": {}} diff --git a/tests/test_github_handler.py b/tests/test_github_handler.py index 8d8fdaa..d0e681a 100644 --- a/tests/test_github_handler.py +++ b/tests/test_github_handler.py @@ -1,30 +1,40 @@ """Tests for datasources/github_handler.py""" -import pytest -from unittest.mock import patch, MagicMock + +from unittest.mock import patch class TestParseGithubEvent: - @patch("datasources.github_handler.fetch_commit_stats", return_value={"total": 10, "additions": 7, "deletions": 3}) + @patch( + "datasources.github_handler.fetch_commit_stats", + return_value={"total": 10, "additions": 7, "deletions": 3}, + ) def test_push_event_dispatches(self, mock_stats, github_push_payload): from datasources.github_handler import parse_github_event + result = parse_github_event(github_push_payload, "TestPrj") assert result["event"] == "commit" assert len(result["commits"]) == 1 def test_issue_event_dispatches(self, github_issue_payload): from datasources.github_handler import parse_github_event + result = parse_github_event(github_issue_payload, "TestPrj") assert result["event"] == "issue" assert result["action"] == "opened" - @patch("datasources.github_handler.to_madrid_local", return_value="2025-06-15T14:00:00.000") + @patch( + "datasources.github_handler.to_madrid_local", + return_value="2025-06-15T14:00:00.000", + ) def test_pull_request_event_dispatches(self, mock_tz, github_pr_payload): from datasources.github_handler import parse_github_event + result = parse_github_event(github_pr_payload, "TestPrj") assert result["event"] == "pull_request" def test_unknown_event_returns_ignored(self): from datasources.github_handler import parse_github_event + payload = {"X-GitHub-Event": "deployment"} result = parse_github_event(payload, "TestPrj") assert result["ignored"] is True @@ -32,9 +42,13 @@ def test_unknown_event_returns_ignored(self): class TestParseGithubPushEvent: - @patch("datasources.github_handler.fetch_commit_stats", return_value={"total": 15, "additions": 10, "deletions": 5}) + @patch( + "datasources.github_handler.fetch_commit_stats", + return_value={"total": 15, "additions": 10, "deletions": 5}, + ) def test_basic_push_parsing(self, mock_stats, github_push_payload): from datasources.github_handler import parse_github_push_event + result = parse_github_push_event(github_push_payload, "TestPrj") assert result["event"] == "commit" @@ -43,9 +57,13 @@ def test_basic_push_parsing(self, mock_stats, github_push_payload): assert result["sender_info"]["login"] == "devuser" assert result["sender_info"]["id"] == 1 - @patch("datasources.github_handler.fetch_commit_stats", return_value={"total": 0, "additions": 0, "deletions": 0}) + @patch( + "datasources.github_handler.fetch_commit_stats", + return_value={"total": 0, "additions": 0, "deletions": 0}, + ) def test_commit_details(self, mock_stats, github_push_payload): from datasources.github_handler import parse_github_push_event + result = parse_github_push_event(github_push_payload, "TestPrj") commit = result["commits"][0] @@ -57,102 +75,132 @@ def test_commit_details(self, mock_stats, github_push_payload): assert commit["message_char_count"] == len("fix: resolve task #42 issue") assert commit["message_word_count"] == 5 - @patch("datasources.github_handler.fetch_commit_stats", return_value={"total": 0, "additions": 0, "deletions": 0}) + @patch( + "datasources.github_handler.fetch_commit_stats", + return_value={"total": 0, "additions": 0, "deletions": 0}, + ) def test_task_reference_with_number(self, mock_stats, github_push_payload): from datasources.github_handler import parse_github_push_event + result = parse_github_push_event(github_push_payload, "TestPrj") commit = result["commits"][0] assert commit["task_is_written"] is True assert commit["task_reference"] == 42 - @patch("datasources.github_handler.fetch_commit_stats", return_value={"total": 0, "additions": 0, "deletions": 0}) + @patch( + "datasources.github_handler.fetch_commit_stats", + return_value={"total": 0, "additions": 0, "deletions": 0}, + ) def test_task_reference_catalan(self, mock_stats): from datasources.github_handler import parse_github_push_event + payload = { "organization": {"login": "Org"}, "repository": {"full_name": "Org/repo"}, "sender": {}, - "commits": [{ - "id": "sha1", - "url": "", - "message": "Implementar tasca #99", - "timestamp": "2025-06-15T10:30:00Z", - "author": {"username": "u", "name": "n", "email": "e"} - }] + "commits": [ + { + "id": "sha1", + "url": "", + "message": "Implementar tasca #99", + "timestamp": "2025-06-15T10:30:00Z", + "author": {"username": "u", "name": "n", "email": "e"}, + } + ], } result = parse_github_push_event(payload, "P") commit = result["commits"][0] assert commit["task_is_written"] is True assert commit["task_reference"] == 99 - @patch("datasources.github_handler.fetch_commit_stats", return_value={"total": 0, "additions": 0, "deletions": 0}) + @patch( + "datasources.github_handler.fetch_commit_stats", + return_value={"total": 0, "additions": 0, "deletions": 0}, + ) def test_task_word_without_number(self, mock_stats): from datasources.github_handler import parse_github_push_event + payload = { "organization": {"login": "Org"}, "repository": {"full_name": "Org/repo"}, "sender": {}, - "commits": [{ - "id": "sha1", - "url": "", - "message": "Working on a task", - "timestamp": "2025-06-15T10:30:00Z", - "author": {"username": "u", "name": "n", "email": "e"} - }] + "commits": [ + { + "id": "sha1", + "url": "", + "message": "Working on a task", + "timestamp": "2025-06-15T10:30:00Z", + "author": {"username": "u", "name": "n", "email": "e"}, + } + ], } result = parse_github_push_event(payload, "P") commit = result["commits"][0] assert commit["task_is_written"] is True assert commit["task_reference"] is None - @patch("datasources.github_handler.fetch_commit_stats", return_value={"total": 0, "additions": 0, "deletions": 0}) + @patch( + "datasources.github_handler.fetch_commit_stats", + return_value={"total": 0, "additions": 0, "deletions": 0}, + ) def test_no_task_reference(self, mock_stats): from datasources.github_handler import parse_github_push_event + payload = { "organization": {"login": "Org"}, "repository": {"full_name": "Org/repo"}, "sender": {}, - "commits": [{ - "id": "sha1", - "url": "", - "message": "fix a bug", - "timestamp": "2025-06-15T10:30:00Z", - "author": {"username": "u", "name": "n", "email": "e"} - }] + "commits": [ + { + "id": "sha1", + "url": "", + "message": "fix a bug", + "timestamp": "2025-06-15T10:30:00Z", + "author": {"username": "u", "name": "n", "email": "e"}, + } + ], } result = parse_github_push_event(payload, "P") commit = result["commits"][0] assert commit["task_is_written"] is False assert commit["task_reference"] is None - @patch("datasources.github_handler.fetch_commit_stats", return_value={"total": 0, "additions": 0, "deletions": 0}) + @patch( + "datasources.github_handler.fetch_commit_stats", + return_value={"total": 0, "additions": 0, "deletions": 0}, + ) def test_empty_commits_list(self, mock_stats): from datasources.github_handler import parse_github_push_event + payload = { "organization": {"login": "Org"}, "repository": {"full_name": "Org/repo"}, "sender": {}, - "commits": [] + "commits": [], } result = parse_github_push_event(payload, "P") assert result["commits"] == [] - @patch("datasources.github_handler.fetch_commit_stats", return_value={"total": 20, "additions": 15, "deletions": 5}) + @patch( + "datasources.github_handler.fetch_commit_stats", + return_value={"total": 20, "additions": 15, "deletions": 5}, + ) def test_commit_stats_stored(self, mock_stats, github_push_payload): from datasources.github_handler import parse_github_push_event + result = parse_github_push_event(github_push_payload, "TestPrj") commit = result["commits"][0] assert commit["stats"] == {"total": 20, "additions": 15, "deletions": 5} - @patch("datasources.github_handler.fetch_commit_stats", return_value={"total": 0, "additions": 0, "deletions": 0}) + @patch( + "datasources.github_handler.fetch_commit_stats", + return_value={"total": 0, "additions": 0, "deletions": 0}, + ) def test_missing_organization(self, mock_stats): from datasources.github_handler import parse_github_push_event - payload = { - "repository": {"full_name": "Org/repo"}, - "sender": {}, - "commits": [] - } + + payload = {"repository": {"full_name": "Org/repo"}, "sender": {}, "commits": []} result = parse_github_push_event(payload, "P") assert result["team_name"] == "UnknownTeam" @@ -160,7 +208,8 @@ def test_missing_organization(self, mock_stats): class TestParseGithubIssueEvent: def test_basic_issue_parsing(self, github_issue_payload): from datasources.github_handler import parse_github_issue_event - result = parse_github_issue_event(github_issue_payload, "TestPrj") + + result = parse_github_issue_event(github_issue_payload) assert result["event"] == "issue" assert result["action"] == "opened" @@ -169,7 +218,8 @@ def test_basic_issue_parsing(self, github_issue_payload): def test_issue_object(self, github_issue_payload): from datasources.github_handler import parse_github_issue_event - result = parse_github_issue_event(github_issue_payload, "TestPrj") + + result = parse_github_issue_event(github_issue_payload) issue = result["issue"] assert issue["number"] == 10 @@ -180,15 +230,20 @@ def test_issue_object(self, github_issue_payload): def test_sender_info(self, github_issue_payload): from datasources.github_handler import parse_github_issue_event - result = parse_github_issue_event(github_issue_payload, "TestPrj") + + result = parse_github_issue_event(github_issue_payload) assert result["sender_info"]["login"] == "issueuser" assert result["sender_info"]["id"] == 2 class TestParseGithubPullRequestEvent: - @patch("datasources.github_handler.to_madrid_local", return_value="2025-06-15T14:00:00.000") + @patch( + "datasources.github_handler.to_madrid_local", + return_value="2025-06-15T14:00:00.000", + ) def test_closed_pr_parsed(self, mock_tz, github_pr_payload): from datasources.github_handler import parse_github_pullrequest_event + result = parse_github_pullrequest_event(github_pr_payload, "TestPrj") assert result["event"] == "pull_request" @@ -200,12 +255,13 @@ def test_closed_pr_parsed(self, mock_tz, github_pr_payload): def test_non_closed_pr_ignored(self): from datasources.github_handler import parse_github_pullrequest_event + payload = { "action": "opened", "organization": {"login": "Org"}, "repository": {"full_name": "Org/repo"}, "sender": {}, - "pull_request": {} + "pull_request": {}, } result = parse_github_pullrequest_event(payload, "P") assert result["ignored"] is True @@ -213,6 +269,7 @@ def test_non_closed_pr_ignored(self): @patch("datasources.github_handler.to_madrid_local", return_value="") def test_pr_no_assignee(self, mock_tz): from datasources.github_handler import parse_github_pullrequest_event + payload = { "action": "closed", "organization": {"login": "Org"}, @@ -226,8 +283,8 @@ def test_pr_no_assignee(self, mock_tz): "merged": False, "merged_by": {}, "assignee": None, - "requested_reviewers": [] - } + "requested_reviewers": [], + }, } result = parse_github_pullrequest_event(payload, "P") assert result["reviewers"] == [] diff --git a/tests/test_github_recovery.py b/tests/test_github_recovery.py index e434a29..f018d5e 100644 --- a/tests/test_github_recovery.py +++ b/tests/test_github_recovery.py @@ -1,12 +1,12 @@ """Tests for utils/recovery/github_recovery.py""" -import pytest + from unittest.mock import patch, MagicMock -from datetime import datetime class TestParseDt: def test_date_only(self): from utils.recovery.github_recovery import parse_dt + result = parse_dt("2025-06-15") assert result is not None assert result.year == 2025 @@ -15,22 +15,26 @@ def test_date_only(self): def test_datetime_with_time(self): from utils.recovery.github_recovery import parse_dt + result = parse_dt("2025-06-15T14:30") assert result.hour == 14 assert result.minute == 30 def test_none_returns_none(self): from utils.recovery.github_recovery import parse_dt + result = parse_dt(None) assert result is None def test_empty_string_returns_none(self): from utils.recovery.github_recovery import parse_dt + result = parse_dt("") assert result is None def test_timezone_aware(self): from utils.recovery.github_recovery import parse_dt + result = parse_dt("2025-06-15") assert result.tzinfo is not None @@ -39,6 +43,7 @@ class TestGetOrganizationRepos: @patch("utils.recovery.github_recovery.gh_paginated") def test_returns_repo_names(self, mock_pag): from utils.recovery.github_recovery import get_organization_repos + mock_pag.return_value = [ {"name": "repo1"}, {"name": "repo2"}, @@ -88,6 +93,7 @@ def test_multiple_pages(self, mock_get): class TestUpsert: def test_upsert_empty_list(self): from utils.recovery.github_recovery import upsert + mock_coll = MagicMock() result = upsert(mock_coll, [], "sha") assert result == 0 @@ -95,6 +101,7 @@ def test_upsert_empty_list(self): def test_upsert_with_docs(self): from utils.recovery.github_recovery import upsert + mock_coll = MagicMock() mock_result = MagicMock() mock_result.matched_count = 1 @@ -115,22 +122,28 @@ class TestCollectGithub: def test_collect_commits(self, mock_pag, mock_parse, mock_coll, mock_notify): from utils.recovery.github_recovery import collect_github - mock_pag.return_value = [{ - "sha": "abc123", - "url": "https://github.com/Org/repo/commit/abc123", - "commit": { - "message": "fix bug", - "author": {"date": "2025-06-15T10:00:00Z", "name": "Dev", "email": "d@e.com"} - }, - "author": {"login": "dev"} - }] + mock_pag.return_value = [ + { + "sha": "abc123", + "url": "https://github.com/Org/repo/commit/abc123", + "commit": { + "message": "fix bug", + "author": { + "date": "2025-06-15T10:00:00Z", + "name": "Dev", + "email": "d@e.com", + }, + }, + "author": {"login": "dev"}, + } + ] mock_parse.return_value = { "event": "commit", "team_name": "Org", "repo_name": "Org/repo", "sender_info": {"login": "dev"}, - "commits": [{"sha": "abc123", "message": "fix bug"}] + "commits": [{"sha": "abc123", "message": "fix bug"}], } mock_collection = MagicMock() @@ -152,19 +165,16 @@ def test_collect_commits(self, mock_pag, mock_parse, mock_coll, mock_notify): def test_collect_issues(self, mock_pag, mock_parse, mock_coll, mock_notify): from utils.recovery.github_recovery import collect_github - mock_pag.return_value = [{ - "number": 1, - "state": "open", - "user": {"login": "dev"}, - "title": "Bug" - }] + mock_pag.return_value = [ + {"number": 1, "state": "open", "user": {"login": "dev"}, "title": "Bug"} + ] mock_parse.return_value = { "event": "issue", "team_name": "Org", "repo_name": "Org/repo", "sender_info": {"login": "dev"}, - "issue": {"number": 1} + "issue": {"number": 1}, } mock_collection = MagicMock() @@ -184,18 +194,14 @@ def test_collect_issues(self, mock_pag, mock_parse, mock_coll, mock_notify): def test_collect_pull_requests(self, mock_pag, mock_parse, mock_coll, mock_notify): from utils.recovery.github_recovery import collect_github - mock_pag.return_value = [{ - "number": 5, - "user": {"login": "dev"}, - "title": "PR" - }] + mock_pag.return_value = [{"number": 5, "user": {"login": "dev"}, "title": "PR"}] mock_parse.return_value = { "event": "pull_request", "team_name": "Org", "repo_name": "Org/repo", "sender_info": {"login": "dev"}, - "pr_number": 5 + "pr_number": 5, } mock_collection = MagicMock() @@ -205,7 +211,9 @@ def test_collect_pull_requests(self, mock_pag, mock_parse, mock_coll, mock_notif mock_collection.bulk_write.return_value = mock_result mock_coll.return_value = mock_collection - collect_github("Org", "repo", "TestPrj", ["pull_requests"], None, None, "default") + collect_github( + "Org", "repo", "TestPrj", ["pull_requests"], None, None, "default" + ) mock_coll.assert_called_with("github_TestPrj.pull_requests") @patch("utils.recovery.github_recovery.notify_eval_push") @@ -213,5 +221,6 @@ def test_collect_pull_requests(self, mock_pag, mock_parse, mock_coll, mock_notif @patch("utils.recovery.github_recovery.gh_paginated") def test_unsupported_event_logged(self, mock_pag, mock_coll, mock_notify): from utils.recovery.github_recovery import collect_github + # Should not crash on unsupported event collect_github("Org", "repo", "P", ["unsupported"], None, None, "default") diff --git a/tests/test_github_routes.py b/tests/test_github_routes.py index 5cec735..7955961 100644 --- a/tests/test_github_routes.py +++ b/tests/test_github_routes.py @@ -1,5 +1,8 @@ """Tests for routes/github_routes.py""" -import hashlib, hmac, json, pytest + +import hashlib +import hmac +import json from unittest.mock import patch, MagicMock @@ -12,15 +15,15 @@ class TestGithubWebhook: @patch("routes.github_routes.get_collection") @patch("routes.github_routes.parse_github_event") @patch("routes.github_routes.verify_github_signature", return_value=True) - def test_push_event_inserts_commits(self, mock_verify, mock_parse, mock_coll, mock_notify, client): + def test_push_event_inserts_commits( + self, mock_verify, mock_parse, mock_coll, mock_notify, client + ): mock_parse.return_value = { "event": "commit", "team_name": "TestOrg", "repo_name": "TestOrg/repo", "sender_info": {"login": "dev"}, - "commits": [ - {"sha": "abc123", "message": "fix bug"} - ] + "commits": [{"sha": "abc123", "message": "fix bug"}], } mock_collection = MagicMock() mock_coll.return_value = mock_collection @@ -30,10 +33,7 @@ def test_push_event_inserts_commits(self, mock_verify, mock_parse, mock_coll, mo "/webhook/github?prj=TestPrj&quality_model=default", data=body, content_type="application/json", - headers={ - "X-GitHub-Event": "push", - "X-Hub-Signature-256": "sha256=fake" - } + headers={"X-GitHub-Event": "push", "X-Hub-Signature-256": "sha256=fake"}, ) assert resp.status_code == 200 mock_collection.insert_one.assert_called_once() @@ -45,7 +45,7 @@ def test_invalid_signature_returns_403(self, mock_verify, client): "/webhook/github?prj=TestPrj", data=json.dumps({"test": True}), content_type="application/json", - headers={"X-Hub-Signature-256": "sha256=bad"} + headers={"X-Hub-Signature-256": "sha256=bad"}, ) assert resp.status_code == 403 @@ -55,7 +55,7 @@ def test_missing_json_returns_400(self, mock_verify, client): "/webhook/github?prj=TestPrj", data="", content_type="application/json", - headers={"X-Hub-Signature-256": "sha256=x"} + headers={"X-Hub-Signature-256": "sha256=x"}, ) assert resp.status_code == 400 @@ -65,21 +65,23 @@ def test_missing_prj_returns_400(self, mock_verify, client): "/webhook/github", data=json.dumps({"test": True}), content_type="application/json", - headers={"X-Hub-Signature-256": "sha256=x"} + headers={"X-Hub-Signature-256": "sha256=x"}, ) assert resp.status_code == 400 @patch("routes.github_routes.get_collection") @patch("routes.github_routes.parse_github_event") @patch("routes.github_routes.verify_github_signature", return_value=True) - def test_ignored_event_returns_200(self, mock_verify, mock_parse, mock_coll, client): + def test_ignored_event_returns_200( + self, mock_verify, mock_parse, mock_coll, client + ): mock_parse.return_value = {"event": "deployment", "ignored": True} resp = client.post( "/webhook/github?prj=TestPrj", data=json.dumps({"test": True}), content_type="application/json", - headers={"X-GitHub-Event": "deployment", "X-Hub-Signature-256": "sha256=x"} + headers={"X-GitHub-Event": "deployment", "X-Hub-Signature-256": "sha256=x"}, ) assert resp.status_code == 200 data = resp.get_json() @@ -89,13 +91,15 @@ def test_ignored_event_returns_200(self, mock_verify, mock_parse, mock_coll, cli @patch("routes.github_routes.get_collection") @patch("routes.github_routes.parse_github_event") @patch("routes.github_routes.verify_github_signature", return_value=True) - def test_issue_event_inserts_doc(self, mock_verify, mock_parse, mock_coll, mock_notify, client): + def test_issue_event_inserts_doc( + self, mock_verify, mock_parse, mock_coll, mock_notify, client + ): mock_parse.return_value = { "event": "issue", "team_name": "TestOrg", "repo_name": "TestOrg/repo", "sender_info": {"login": "dev"}, - "issue": {"number": 1, "title": "Bug"} + "issue": {"number": 1, "title": "Bug"}, } mock_collection = MagicMock() mock_coll.return_value = mock_collection @@ -104,7 +108,7 @@ def test_issue_event_inserts_doc(self, mock_verify, mock_parse, mock_coll, mock_ "/webhook/github?prj=TestPrj", data=json.dumps({"test": True}), content_type="application/json", - headers={"X-GitHub-Event": "issues", "X-Hub-Signature-256": "sha256=x"} + headers={"X-GitHub-Event": "issues", "X-Hub-Signature-256": "sha256=x"}, ) assert resp.status_code == 200 mock_collection.insert_one.assert_called_once() @@ -113,13 +117,15 @@ def test_issue_event_inserts_doc(self, mock_verify, mock_parse, mock_coll, mock_ @patch("routes.github_routes.get_collection") @patch("routes.github_routes.parse_github_event") @patch("routes.github_routes.verify_github_signature", return_value=True) - def test_pull_request_event(self, mock_verify, mock_parse, mock_coll, mock_notify, client): + def test_pull_request_event( + self, mock_verify, mock_parse, mock_coll, mock_notify, client + ): mock_parse.return_value = { "event": "pull_request", "team_name": "TestOrg", "repo_name": "TestOrg/repo", "sender_info": {"login": "dev"}, - "pull_request": {"number": 5} + "pull_request": {"number": 5}, } mock_collection = MagicMock() mock_coll.return_value = mock_collection @@ -128,7 +134,10 @@ def test_pull_request_event(self, mock_verify, mock_parse, mock_coll, mock_notif "/webhook/github?prj=TestPrj", data=json.dumps({"test": True}), content_type="application/json", - headers={"X-GitHub-Event": "pull_request", "X-Hub-Signature-256": "sha256=x"} + headers={ + "X-GitHub-Event": "pull_request", + "X-Hub-Signature-256": "sha256=x", + }, ) assert resp.status_code == 200 @@ -141,7 +150,7 @@ def test_error_in_parsed_data(self, mock_verify, mock_parse, client): "/webhook/github?prj=TestPrj", data=json.dumps({"test": True}), content_type="application/json", - headers={"X-GitHub-Event": "push", "X-Hub-Signature-256": "sha256=x"} + headers={"X-GitHub-Event": "push", "X-Hub-Signature-256": "sha256=x"}, ) assert resp.status_code == 400 @@ -149,13 +158,15 @@ def test_error_in_parsed_data(self, mock_verify, mock_parse, client): @patch("routes.github_routes.get_collection") @patch("routes.github_routes.parse_github_event") @patch("routes.github_routes.verify_github_signature", return_value=True) - def test_notify_eval_error_returns_500(self, mock_verify, mock_parse, mock_coll, mock_notify, client): + def test_notify_eval_error_returns_500( + self, mock_verify, mock_parse, mock_coll, mock_notify, client + ): mock_parse.return_value = { "event": "commit", "team_name": "T", "repo_name": "T/r", "sender_info": {"login": "u"}, - "commits": [{"sha": "a"}] + "commits": [{"sha": "a"}], } mock_coll.return_value = MagicMock() @@ -163,7 +174,7 @@ def test_notify_eval_error_returns_500(self, mock_verify, mock_parse, mock_coll, "/webhook/github?prj=P", data=json.dumps({"test": True}), content_type="application/json", - headers={"X-GitHub-Event": "push", "X-Hub-Signature-256": "sha256=x"} + headers={"X-GitHub-Event": "push", "X-Hub-Signature-256": "sha256=x"}, ) assert resp.status_code == 500 @@ -171,13 +182,15 @@ def test_notify_eval_error_returns_500(self, mock_verify, mock_parse, mock_coll, @patch("routes.github_routes.get_collection") @patch("routes.github_routes.parse_github_event") @patch("routes.github_routes.verify_github_signature", return_value=True) - def test_collection_name_commit(self, mock_verify, mock_parse, mock_coll, mock_notify, client): + def test_collection_name_commit( + self, mock_verify, mock_parse, mock_coll, mock_notify, client + ): mock_parse.return_value = { "event": "commit", "team_name": "T", "repo_name": "T/r", "sender_info": {"login": "u"}, - "commits": [{"sha": "a"}] + "commits": [{"sha": "a"}], } mock_coll.return_value = MagicMock() @@ -185,7 +198,7 @@ def test_collection_name_commit(self, mock_verify, mock_parse, mock_coll, mock_n "/webhook/github?prj=MyPrj", data=json.dumps({"test": True}), content_type="application/json", - headers={"X-GitHub-Event": "push", "X-Hub-Signature-256": "sha256=x"} + headers={"X-GitHub-Event": "push", "X-Hub-Signature-256": "sha256=x"}, ) mock_coll.assert_called_with("github_MyPrj.commits") @@ -193,13 +206,15 @@ def test_collection_name_commit(self, mock_verify, mock_parse, mock_coll, mock_n @patch("routes.github_routes.get_collection") @patch("routes.github_routes.parse_github_event") @patch("routes.github_routes.verify_github_signature", return_value=True) - def test_collection_name_issue(self, mock_verify, mock_parse, mock_coll, mock_notify, client): + def test_collection_name_issue( + self, mock_verify, mock_parse, mock_coll, mock_notify, client + ): mock_parse.return_value = { "event": "issue", "team_name": "T", "repo_name": "T/r", "sender_info": {"login": "u"}, - "issue": {"number": 1} + "issue": {"number": 1}, } mock_coll.return_value = MagicMock() @@ -207,6 +222,6 @@ def test_collection_name_issue(self, mock_verify, mock_parse, mock_coll, mock_no "/webhook/github?prj=MyPrj", data=json.dumps({"test": True}), content_type="application/json", - headers={"X-GitHub-Event": "issues", "X-Hub-Signature-256": "sha256=x"} + headers={"X-GitHub-Event": "issues", "X-Hub-Signature-256": "sha256=x"}, ) mock_coll.assert_called_with("MyPrj_issues") diff --git a/tests/test_logger_config.py b/tests/test_logger_config.py index 47f512e..5eeca78 100644 --- a/tests/test_logger_config.py +++ b/tests/test_logger_config.py @@ -1,5 +1,7 @@ """Tests for config/logger_config.py""" -import logging, os, pytest + +import logging +import os from unittest.mock import patch @@ -12,6 +14,7 @@ def test_setup_logging_default_level(self): with patch.dict(os.environ, {}, clear=False): os.environ.pop("LOG_LEVEL", None) from config.logger_config import setup_logging + setup_logging() assert root.level == logging.INFO @@ -20,6 +23,7 @@ def test_setup_logging_custom_level(self): root.handlers.clear() with patch.dict(os.environ, {"LOG_LEVEL": "DEBUG"}): from config.logger_config import setup_logging + setup_logging() assert root.level == logging.DEBUG @@ -28,6 +32,7 @@ def test_setup_logging_idempotent(self): root = logging.getLogger() root.handlers.clear() from config.logger_config import setup_logging + setup_logging() count_after_first = len(root.handlers) setup_logging() @@ -38,6 +43,7 @@ def test_setup_logging_invalid_level_falls_back(self): root.handlers.clear() with patch.dict(os.environ, {"LOG_LEVEL": "INVALID_LEVEL"}): from config.logger_config import setup_logging + setup_logging() # Should fall back to INFO assert root.level == logging.INFO diff --git a/tests/test_mongo_client.py b/tests/test_mongo_client.py index 85d6fae..00a7156 100644 --- a/tests/test_mongo_client.py +++ b/tests/test_mongo_client.py @@ -1,5 +1,5 @@ """Tests for database/mongo_client.py""" -import pytest + from unittest.mock import patch, MagicMock @@ -11,6 +11,7 @@ def test_get_collection_returns_collection(self): with patch("database.mongo_client.db", mock_db): from database.mongo_client import get_collection + result = get_collection("test_collection") mock_db.__getitem__.assert_called_once_with("test_collection") assert result == mock_collection @@ -28,6 +29,7 @@ def side_effect(name): with patch("database.mongo_client.db", mock_db): from database.mongo_client import get_collection + c1 = get_collection("commits") c2 = get_collection("issues") assert c1 != c2 diff --git a/tests/test_settings.py b/tests/test_settings.py index d4ab09f..dc84976 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1,5 +1,9 @@ """Tests for config/settings.py""" -import os, importlib, pytest + +import os +import importlib +import pytest +from pathlib import Path from unittest.mock import patch import config.settings as settings_mod @@ -13,10 +17,9 @@ def test_require_env_missing_raises(self): "TAIGA_SIGNATURE_KEY": "x", "TAIGA_USERNAME": "x", "TAIGA_PASSWORD": "x", - "HOME": os.environ.get("HOME", "/tmp"), + "HOME": os.environ.get("HOME") or str(Path.home()), } - with patch.dict(os.environ, env, clear=True), \ - patch("dotenv.load_dotenv"): + with patch.dict(os.environ, env, clear=True), patch("dotenv.load_dotenv"): with pytest.raises(RuntimeError, match="TAIGA_API_URL"): importlib.reload(settings_mod) # Restore module to working state @@ -38,6 +41,7 @@ def test_mongo_uri_without_credentials(self): with patch.dict(os.environ, env, clear=True): import importlib import config.settings as settings_mod + importlib.reload(settings_mod) assert settings_mod.MONGO_URI == "mongodb://myhost:27017/mydb" @@ -58,6 +62,7 @@ def test_mongo_uri_with_credentials(self): with patch.dict(os.environ, env, clear=True): import importlib import config.settings as settings_mod + importlib.reload(settings_mod) assert "admin:secret" in settings_mod.MONGO_URI assert "authSource=authdb" in settings_mod.MONGO_URI @@ -73,6 +78,7 @@ def test_default_values(self): with patch.dict(os.environ, env, clear=True): import importlib import config.settings as settings_mod + importlib.reload(settings_mod) assert settings_mod.MONGO_HOST == "mongodb" assert settings_mod.MONGO_PORT == "27017" @@ -85,9 +91,8 @@ def test_taiga_auth_url_defaults_to_api_url(self): "TAIGA_SIGNATURE_KEY": "ts", "TAIGA_USERNAME": "u", "TAIGA_PASSWORD": "p", - "HOME": os.environ.get("HOME", "/tmp"), + "HOME": os.environ.get("HOME") or str(Path.home()), } - with patch.dict(os.environ, env, clear=True), \ - patch("dotenv.load_dotenv"): + with patch.dict(os.environ, env, clear=True), patch("dotenv.load_dotenv"): importlib.reload(settings_mod) assert settings_mod.TAIGA_AUTH_URL == "https://custom.taiga.io/api/v1" diff --git a/tests/test_taiga_api_call.py b/tests/test_taiga_api_call.py index 70ae180..b73965c 100644 --- a/tests/test_taiga_api_call.py +++ b/tests/test_taiga_api_call.py @@ -1,17 +1,24 @@ """Tests for datasources/requests/taiga_api_call.py""" -import pytest + from unittest.mock import patch, MagicMock -from datetime import datetime, timedelta class TestMilestoneStats: def setup_method(self): """Clear the cache before each test.""" import datasources.requests.taiga_api_call as mod + mod._CACHE.clear() - @patch("datasources.requests.taiga_api_call.resolve", side_effect=lambda prj, f: {"taiga_user": "u", "taiga_password": "p"}.get(f, "")) - @patch("datasources.requests.taiga_api_call.get_taiga_token", return_value="fake_token") + @patch( + "datasources.requests.taiga_api_call.resolve", + side_effect=lambda prj, f: {"taiga_user": "u", "taiga_password": "p"}.get( + f, "" + ), + ) + @patch( + "datasources.requests.taiga_api_call.get_taiga_token", return_value="fake_token" + ) @patch("datasources.requests.taiga_api_call.requests.get") def test_successful_fetch(self, mock_get, mock_token, mock_resolve): from datasources.requests.taiga_api_call import milestone_stats @@ -23,7 +30,7 @@ def test_successful_fetch(self, mock_get, mock_token, mock_resolve): "total_userstories": 4, "completed_userstories": 2, "total_tasks": 10, - "completed_tasks": 6 + "completed_tasks": 6, } mock_resp.raise_for_status = MagicMock() mock_get.return_value = mock_resp @@ -38,16 +45,25 @@ def test_successful_fetch(self, mock_get, mock_token, mock_resolve): def test_empty_project_id(self): from datasources.requests.taiga_api_call import milestone_stats + result = milestone_stats("", "mile1", "P") assert result == {} def test_empty_milestone_id(self): from datasources.requests.taiga_api_call import milestone_stats + result = milestone_stats("proj1", "", "P") assert result == {} - @patch("datasources.requests.taiga_api_call.resolve", side_effect=lambda prj, f: {"taiga_user": "u", "taiga_password": "p"}.get(f, "")) - @patch("datasources.requests.taiga_api_call.get_taiga_token", return_value="fake_token") + @patch( + "datasources.requests.taiga_api_call.resolve", + side_effect=lambda prj, f: {"taiga_user": "u", "taiga_password": "p"}.get( + f, "" + ), + ) + @patch( + "datasources.requests.taiga_api_call.get_taiga_token", return_value="fake_token" + ) @patch("datasources.requests.taiga_api_call.requests.get") def test_caching(self, mock_get, mock_token, mock_resolve): from datasources.requests.taiga_api_call import milestone_stats @@ -59,7 +75,7 @@ def test_caching(self, mock_get, mock_token, mock_resolve): "total_userstories": 1, "completed_userstories": 0, "total_tasks": 1, - "completed_tasks": 0 + "completed_tasks": 0, } mock_resp.raise_for_status = MagicMock() mock_get.return_value = mock_resp @@ -70,7 +86,10 @@ def test_caching(self, mock_get, mock_token, mock_resolve): milestone_stats("p1", "m1", "P") assert mock_get.call_count == 1 # Only called once due to caching - @patch("datasources.requests.taiga_api_call.resolve", side_effect=lambda prj, f: {"taiga_user": "", "taiga_password": ""}.get(f, "")) + @patch( + "datasources.requests.taiga_api_call.resolve", + side_effect=lambda prj, f: {"taiga_user": "", "taiga_password": ""}.get(f, ""), + ) @patch("datasources.requests.taiga_api_call.requests.get") def test_no_credentials_no_auth_header(self, mock_get, mock_resolve): from datasources.requests.taiga_api_call import milestone_stats @@ -82,7 +101,7 @@ def test_no_credentials_no_auth_header(self, mock_get, mock_resolve): "total_userstories": 0, "completed_userstories": 0, "total_tasks": 0, - "completed_tasks": 0 + "completed_tasks": 0, } mock_resp.raise_for_status = MagicMock() mock_get.return_value = mock_resp @@ -91,7 +110,12 @@ def test_no_credentials_no_auth_header(self, mock_get, mock_resolve): headers = mock_get.call_args[1]["headers"] assert "Authorization" not in headers - @patch("datasources.requests.taiga_api_call.resolve", side_effect=lambda prj, f: {"taiga_user": "u", "taiga_password": "p"}.get(f, "")) + @patch( + "datasources.requests.taiga_api_call.resolve", + side_effect=lambda prj, f: {"taiga_user": "u", "taiga_password": "p"}.get( + f, "" + ), + ) @patch("datasources.requests.taiga_api_call.get_taiga_token", return_value="tok") @patch("datasources.requests.taiga_api_call.requests.get") def test_http_error_returns_zeros(self, mock_get, mock_token, mock_resolve): diff --git a/tests/test_taiga_auth.py b/tests/test_taiga_auth.py index 7a2f53c..6b24113 100644 --- a/tests/test_taiga_auth.py +++ b/tests/test_taiga_auth.py @@ -1,5 +1,7 @@ """Tests for utils/taiga_token/taiga_auth.py""" -import time, pytest + +import time +import pytest from unittest.mock import patch, MagicMock @@ -7,6 +9,7 @@ class TestGetTaigaToken: def setup_method(self): """Clear the token cache before each test.""" import utils.taiga_token.taiga_auth as mod + mod._TOKENS.clear() @patch("utils.taiga_token.taiga_auth.requests.post") diff --git a/tests/test_taiga_handler.py b/tests/test_taiga_handler.py index 125f0ed..1a06293 100644 --- a/tests/test_taiga_handler.py +++ b/tests/test_taiga_handler.py @@ -1,38 +1,44 @@ """Tests for datasources/taiga_handler.py""" -import pytest -from unittest.mock import patch, MagicMock + +from unittest.mock import patch class TestParseTaigaEvent: @patch("datasources.taiga_handler.milestone_stats", return_value={}) def test_task_event(self, mock_ms, taiga_task_payload): from datasources.taiga_handler import parse_taiga_event + result = parse_taiga_event(taiga_task_payload, "TestPrj") assert result["event_type"] == "task" def test_issue_event(self, taiga_issue_payload): from datasources.taiga_handler import parse_taiga_event + result = parse_taiga_event(taiga_issue_payload, "TestPrj") assert result["event_type"] == "issue" def test_epic_event(self, taiga_epic_payload): from datasources.taiga_handler import parse_taiga_event + result = parse_taiga_event(taiga_epic_payload, "TestPrj") assert result["event_type"] == "epic" @patch("datasources.taiga_handler.milestone_stats", return_value={}) def test_userstory_event(self, mock_ms, taiga_userstory_payload): from datasources.taiga_handler import parse_taiga_event + result = parse_taiga_event(taiga_userstory_payload, "TestPrj") assert result["event_type"] == "userstory" def test_relateduserstory_event(self, taiga_related_userstory_payload): from datasources.taiga_handler import parse_taiga_event + result = parse_taiga_event(taiga_related_userstory_payload, "TestPrj") assert result["event_type"] == "relateduserstory" def test_unsupported_event(self): from datasources.taiga_handler import parse_taiga_event + payload = {"type": "unknown_type"} result = parse_taiga_event(payload, "P") assert result["event"] == "unknown_type" @@ -42,7 +48,8 @@ def test_unsupported_event(self): class TestParseTaigaIssueEvent: def test_basic_issue_parsing(self, taiga_issue_payload): from datasources.taiga_handler import parse_taiga_issue_event - result = parse_taiga_issue_event(taiga_issue_payload, "TestPrj") + + result = parse_taiga_issue_event(taiga_issue_payload) assert result["event_type"] == "issue" assert result["action_type"] == "create" @@ -58,6 +65,7 @@ def test_basic_issue_parsing(self, taiga_issue_payload): def test_issue_assigned_to_none(self): from datasources.taiga_handler import parse_taiga_issue_event + payload = { "type": "issue", "action": "create", @@ -75,17 +83,18 @@ def test_issue_assigned_to_none(self): "modified_date": "", "created_date": "", "finished_date": "", - "assigned_to": None + "assigned_to": None, }, - "is_closed": False + "is_closed": False, } - result = parse_taiga_issue_event(payload, "P") + result = parse_taiga_issue_event(payload) assert result["assigned_to"] is None class TestParseTaigaEpicEvent: def test_basic_epic_parsing(self, taiga_epic_payload): from datasources.taiga_handler import parse_taiga_epic_event + result = parse_taiga_epic_event(taiga_epic_payload, "TestPrj") assert result["epic_id"] == 300 @@ -97,9 +106,13 @@ def test_basic_epic_parsing(self, taiga_epic_payload): class TestParseTaigaTaskEvent: - @patch("datasources.taiga_handler.milestone_stats", return_value={"milestone_total_points": 10}) + @patch( + "datasources.taiga_handler.milestone_stats", + return_value={"milestone_total_points": 10}, + ) def test_basic_task_parsing(self, mock_ms, taiga_task_payload): from datasources.taiga_handler import parse_taiga_task_event + result = parse_taiga_task_event(taiga_task_payload, "TestPrj") assert result["event_type"] == "task" @@ -118,6 +131,7 @@ def test_basic_task_parsing(self, mock_ms, taiga_task_payload): @patch("datasources.taiga_handler.milestone_stats", return_value={}) def test_task_assigned_to_none(self, mock_ms): from datasources.taiga_handler import parse_taiga_task_event + payload = { "type": "task", "action": "create", @@ -132,12 +146,18 @@ def test_task_assigned_to_none(self, mock_ms): "modified_date": "", "finished_date": "", "ref": 1, - "milestone": {"id": 1, "name": "S1", "closed": False, - "created_date": "", "modified_date": "", - "estimated_start": "", "estimated_finish": ""}, + "milestone": { + "id": 1, + "name": "S1", + "closed": False, + "created_date": "", + "modified_date": "", + "estimated_start": "", + "estimated_finish": "", + }, "assigned_to": None, - "custom_attributes_values": None - } + "custom_attributes_values": None, + }, } result = parse_taiga_task_event(payload, "P") assert result["assigned_to"] is None @@ -145,9 +165,13 @@ def test_task_assigned_to_none(self, mock_ms): class TestParseTaigaUserstoryEvent: - @patch("datasources.taiga_handler.milestone_stats", return_value={"milestone_total_points": 20}) + @patch( + "datasources.taiga_handler.milestone_stats", + return_value={"milestone_total_points": 20}, + ) def test_basic_userstory_parsing(self, mock_ms, taiga_userstory_payload): from datasources.taiga_handler import parse_taiga_userstory_event + result = parse_taiga_userstory_event(taiga_userstory_payload, "TestPrj") assert result["event_type"] == "userstory" @@ -161,6 +185,7 @@ def test_basic_userstory_parsing(self, mock_ms, taiga_userstory_payload): @patch("datasources.taiga_handler.milestone_stats", return_value={}) def test_userstory_no_pattern(self, mock_ms): from datasources.taiga_handler import parse_taiga_userstory_event + payload = { "type": "userstory", "action": "create", @@ -175,9 +200,9 @@ def test_userstory_no_pattern(self, mock_ms): "description": "Just a simple description", "custom_attributes_values": {}, "points": [], - "milestone": None + "milestone": None, }, - "is_closed": False + "is_closed": False, } result = parse_taiga_userstory_event(payload, "P") assert result["pattern"] is False @@ -189,6 +214,7 @@ def test_userstory_custom_attributes_none(self, mock_ms): """When custom_attributes_values is None, the code crashes on .get('Priority'). This is a known bug in the source. Test with empty dict instead.""" from datasources.taiga_handler import parse_taiga_userstory_event + payload = { "type": "userstory", "action": "create", @@ -203,9 +229,9 @@ def test_userstory_custom_attributes_none(self, mock_ms): "description": "", "custom_attributes_values": {}, "points": [{"value": None}, {"value": 3}], - "milestone": None + "milestone": None, }, - "is_closed": False + "is_closed": False, } result = parse_taiga_userstory_event(payload, "P") assert result["custom_attributes"] == {} @@ -213,10 +239,10 @@ def test_userstory_custom_attributes_none(self, mock_ms): assert result["priority"] == "" @patch("datasources.taiga_handler.milestone_stats", return_value={}) - @pytest.mark.xfail(reason="parse_taiga_userstory_event crashes when custom_attributes_values is None", strict=False) - def test_userstory_custom_attributes_values_none_xfail(self, mock_ms): - """Explicitly exercise the None custom_attributes_values case to capture the known bug.""" + def test_userstory_custom_attributes_values_none(self, mock_ms): + """Ensure None custom_attributes_values is handled without crashing.""" from datasources.taiga_handler import parse_taiga_userstory_event + payload = { "type": "userstory", "action": "create", @@ -231,16 +257,24 @@ def test_userstory_custom_attributes_values_none_xfail(self, mock_ms): "description": "", "custom_attributes_values": None, "points": [{"value": None}, {"value": 3}], - "milestone": None + "milestone": None, }, - "is_closed": False + "is_closed": False, } - with pytest.raises(AttributeError): - parse_taiga_userstory_event(payload, "P") + result = parse_taiga_userstory_event(payload, "P") + + assert result["custom_attributes"] == {} + assert result["priority"] == "" + assert result["total_points"] == 3 + + class TestParseTaigaRelatedUserstoryEvent: def test_basic_related_userstory_parsing(self, taiga_related_userstory_payload): from datasources.taiga_handler import parse_taiga_related_userstory_event - result = parse_taiga_related_userstory_event(taiga_related_userstory_payload, "TestPrj") + + result = parse_taiga_related_userstory_event( + taiga_related_userstory_payload, "TestPrj" + ) assert result["event_type"] == "relateduserstory" assert result["id"] == 400 diff --git a/tests/test_taiga_recovery.py b/tests/test_taiga_recovery.py index 5bb7d38..ab8c3d4 100644 --- a/tests/test_taiga_recovery.py +++ b/tests/test_taiga_recovery.py @@ -1,4 +1,5 @@ """Tests for utils/recovery/taiga_recovery.py""" + import pytest from unittest.mock import patch, MagicMock @@ -6,6 +7,7 @@ class TestParseDt: def test_date_only(self): from utils.recovery.taiga_recovery import parse_dt + result = parse_dt("2025-06-15") assert result.year == 2025 assert result.month == 6 @@ -13,6 +15,7 @@ def test_date_only(self): def test_datetime_with_time(self): from utils.recovery.taiga_recovery import parse_dt + result = parse_dt("2025-06-15T14:30") assert result.hour == 14 assert result.minute == 30 @@ -22,6 +25,7 @@ class TestGetProjectIdBySlug: @patch("utils.recovery.taiga_recovery.requests.get") def test_success(self, mock_get): from utils.recovery.taiga_recovery import get_project_id_by_slug + mock_resp = MagicMock() mock_resp.status_code = 200 mock_resp.json.return_value = {"id": 42} @@ -33,6 +37,7 @@ def test_success(self, mock_get): @patch("utils.recovery.taiga_recovery.requests.get") def test_private_project_exits(self, mock_get): from utils.recovery.taiga_recovery import get_project_id_by_slug + mock_resp = MagicMock() mock_resp.status_code = 401 mock_get.return_value = mock_resp @@ -43,6 +48,7 @@ def test_private_project_exits(self, mock_get): @patch("utils.recovery.taiga_recovery.requests.get") def test_not_found_exits(self, mock_get): from utils.recovery.taiga_recovery import get_project_id_by_slug + mock_resp = MagicMock() mock_resp.status_code = 404 mock_get.return_value = mock_resp @@ -54,6 +60,7 @@ def test_not_found_exits(self, mock_get): def test_other_error_raises(self, mock_get): import requests from utils.recovery.taiga_recovery import get_project_id_by_slug + mock_resp = MagicMock() mock_resp.status_code = 500 mock_resp.raise_for_status.side_effect = requests.HTTPError("500") @@ -67,6 +74,7 @@ class TestGetUsernameId: @patch("utils.recovery.taiga_recovery.requests.get") def test_returns_id(self, mock_get): from utils.recovery.taiga_recovery import get_username_id + mock_resp = MagicMock() mock_resp.json.return_value = {"id": 123} mock_resp.raise_for_status = MagicMock() @@ -81,10 +89,11 @@ class TestGetProjectIdByUsernameId: @patch("utils.recovery.taiga_recovery.requests.get") def test_project_found(self, mock_get, mock_uid): from utils.recovery.taiga_recovery import get_project_id_by_username_id + mock_resp = MagicMock() mock_resp.json.return_value = [ {"name": "Other Project", "id": 10}, - {"name": "My Project", "id": 42} + {"name": "My Project", "id": 42}, ] mock_resp.raise_for_status = MagicMock() mock_get.return_value = mock_resp @@ -96,6 +105,7 @@ def test_project_found(self, mock_get, mock_uid): @patch("utils.recovery.taiga_recovery.requests.get") def test_project_not_found_exits(self, mock_get, mock_uid): from utils.recovery.taiga_recovery import get_project_id_by_username_id + mock_resp = MagicMock() mock_resp.json.return_value = [] mock_resp.raise_for_status = MagicMock() @@ -109,6 +119,7 @@ class TestFetchEntities: @patch("utils.recovery.taiga_recovery.requests.get") def test_fetch_tasks(self, mock_get): from utils.recovery.taiga_recovery import fetch_entities + mock_resp = MagicMock() mock_resp.json.return_value = [{"id": 1}, {"id": 2}] mock_resp.raise_for_status = MagicMock() @@ -119,12 +130,14 @@ def test_fetch_tasks(self, mock_get): def test_unsupported_entity_raises(self): from utils.recovery.taiga_recovery import fetch_entities + with pytest.raises(ValueError, match="Not supported"): fetch_entities("wiki", 42) @patch("utils.recovery.taiga_recovery.requests.get") def test_fetch_with_date_filters(self, mock_get): from utils.recovery.taiga_recovery import fetch_entities, parse_dt + mock_resp = MagicMock() mock_resp.json.return_value = [] mock_resp.raise_for_status = MagicMock() @@ -142,11 +155,13 @@ def test_fetch_with_date_filters(self, mock_get): class TestUpsert: def test_empty(self): from utils.recovery.taiga_recovery import upsert + result = upsert(MagicMock(), [], "id") assert result == 0 def test_with_docs(self): from utils.recovery.taiga_recovery import upsert + mock_coll = MagicMock() mock_result = MagicMock() mock_result.matched_count = 2 @@ -160,6 +175,7 @@ def test_with_docs(self): class TestConverters: def test_task_from_api(self): from utils.recovery.taiga_recovery import task_from_api + j = { "id": 100, "subject": "Task A", @@ -174,7 +190,7 @@ def test_task_from_api(self): "status_extra_info": {"is_closed": False, "name": "New"}, "assigned_to_extra_info": {"username": "dev1"}, "custom_attributes_values": {"sp": 5}, - "project_extra_info": {"name": "TestProject", "id": 1} + "project_extra_info": {"name": "TestProject", "id": 1}, } doc = task_from_api(j, "TestPrj") assert doc["task_id"] == 100 @@ -185,6 +201,7 @@ def test_task_from_api(self): def test_issue_from_api(self): from utils.recovery.taiga_recovery import issue_from_api + j = { "id": 200, "subject": "Bug", @@ -198,7 +215,7 @@ def test_issue_from_api(self): "severity_extra_info": {"name": "Normal"}, "type_extra_info": {"name": "Bug"}, "assigned_to_extra_info": None, - "project_extra_info": {"name": "TestProject"} + "project_extra_info": {"name": "TestProject"}, } doc = issue_from_api(j, "TestPrj") assert doc["issue_id"] == 200 @@ -207,13 +224,14 @@ def test_issue_from_api(self): def test_epic_from_api(self): from utils.recovery.taiga_recovery import epic_from_api + j = { "id": 300, "subject": "Epic", "created_date": "2025-06-01", "modified_date": "2025-06-10", "status_extra_info": {"is_closed": False, "name": "New"}, - "project_extra_info": {"name": "TestProject", "id": 1} + "project_extra_info": {"name": "TestProject", "id": 1}, } doc = epic_from_api(j, "TestPrj") assert doc["epic_id"] == 300 @@ -221,6 +239,7 @@ def test_epic_from_api(self): def test_userstory_from_api(self): from utils.recovery.taiga_recovery import userstory_from_api + j = { "id": 400, "subject": "User login", @@ -229,14 +248,17 @@ def test_userstory_from_api(self): "modified_date": "2025-06-10", "status_extra_info": {"is_closed": False, "name": "New"}, "milestone": 10, - "milestone_extra_info": {"name": "Sprint 1", "closed": False, - "created_date": "2025-05-01", - "modified_date": "2025-06-01", - "estimated_start": "2025-05-01", - "estimated_finish": "2025-06-01"}, + "milestone_extra_info": { + "name": "Sprint 1", + "closed": False, + "created_date": "2025-05-01", + "modified_date": "2025-06-01", + "estimated_start": "2025-05-01", + "estimated_finish": "2025-06-01", + }, "custom_attributes_values": {"Priority": "High"}, "points": [{"value": 3}, {"value": 5}], - "project_extra_info": {"name": "TestProject"} + "project_extra_info": {"name": "TestProject"}, } doc = userstory_from_api(j, "TestPrj") assert doc["userstory_id"] == 400 @@ -246,6 +268,7 @@ def test_userstory_from_api(self): def test_userstory_no_pattern(self): from utils.recovery.taiga_recovery import userstory_from_api + j = { "id": 401, "subject": "S", @@ -257,7 +280,7 @@ def test_userstory_no_pattern(self): "milestone_extra_info": None, "custom_attributes_values": None, "points": "", - "project_extra_info": {"name": "P"} + "project_extra_info": {"name": "P"}, } doc = userstory_from_api(j, "P") assert doc["pattern"] is False @@ -271,8 +294,11 @@ class TestMain: @patch("utils.recovery.taiga_recovery.get_collection") @patch("utils.recovery.taiga_recovery.fetch_entities", return_value=[]) @patch("utils.recovery.taiga_recovery.get_project_id_by_slug", return_value=42) - def test_main_runs(self, mock_slug, mock_fetch, mock_coll, mock_upsert, mock_notify): + def test_main_runs( + self, mock_slug, mock_fetch, mock_coll, mock_upsert, mock_notify + ): from utils.recovery.taiga_recovery import main + main(["--slug", "test-slug", "--prj", "TestPrj", "--events", "task"]) mock_slug.assert_called_once_with("test-slug") mock_fetch.assert_called_once() @@ -282,7 +308,21 @@ def test_main_runs(self, mock_slug, mock_fetch, mock_coll, mock_upsert, mock_not @patch("utils.recovery.taiga_recovery.get_collection") @patch("utils.recovery.taiga_recovery.fetch_entities", return_value=[]) @patch("utils.recovery.taiga_recovery.get_project_id_by_slug", return_value=42) - def test_main_with_dates(self, mock_slug, mock_fetch, mock_coll, mock_upsert, mock_notify): + def test_main_with_dates( + self, mock_slug, mock_fetch, mock_coll, mock_upsert, mock_notify + ): from utils.recovery.taiga_recovery import main - main(["--slug", "s", "--prj", "P", "--from-date", "2025-01-01", "--to-date", "2025-12-31"]) + + main( + [ + "--slug", + "s", + "--prj", + "P", + "--from-date", + "2025-01-01", + "--to-date", + "2025-12-31", + ] + ) mock_fetch.assert_called() diff --git a/tests/test_taiga_routes.py b/tests/test_taiga_routes.py index 0999f72..0d69581 100644 --- a/tests/test_taiga_routes.py +++ b/tests/test_taiga_routes.py @@ -1,5 +1,8 @@ """Tests for routes/taiga_routes.py""" -import hashlib, hmac, json, pytest + +import hashlib +import hmac +import json from unittest.mock import patch, MagicMock @@ -12,19 +15,27 @@ def _post(self, client, payload, prj="TestPrj", quality_model="default"): f"/webhook/taiga?prj={prj}&quality_model={quality_model}", data=body, content_type="application/json", - headers={"X-TAIGA-WEBHOOK-SIGNATURE": sig} + headers={"X-TAIGA-WEBHOOK-SIGNATURE": sig}, ) @patch("routes.taiga_routes.notify_eval_push") @patch("routes.taiga_routes.get_collection") @patch("routes.taiga_routes.parse_taiga_event") @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) - def test_task_create_upserts(self, mock_verify, mock_parse, mock_coll, mock_notify, client, taiga_task_payload): + def test_task_create_upserts( + self, + mock_verify, + mock_parse, + mock_coll, + mock_notify, + client, + taiga_task_payload, + ): mock_parse.return_value = { "event_type": "task", "task_id": 100, "assigned_by": "u", - "subject": "S" + "subject": "S", } mock_collection = MagicMock() mock_coll.return_value = mock_collection @@ -40,7 +51,7 @@ def test_invalid_signature_403(self, mock_verify, client, taiga_task_payload): "/webhook/taiga?prj=P", data=json.dumps(taiga_task_payload), content_type="application/json", - headers={"X-TAIGA-WEBHOOK-SIGNATURE": "bad"} + headers={"X-TAIGA-WEBHOOK-SIGNATURE": "bad"}, ) assert resp.status_code == 403 @@ -50,18 +61,22 @@ def test_missing_json_400(self, mock_verify, client): "/webhook/taiga?prj=P", data="", content_type="application/json", - headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"} + headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"}, ) assert resp.status_code == 400 @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) def test_unsupported_type_ignored(self, mock_verify, client): - payload = {"type": "wiki", "action": "create", "data": {"id": 1, "project": {"name": "P"}}} + payload = { + "type": "wiki", + "action": "create", + "data": {"id": 1, "project": {"name": "P"}}, + } resp = client.post( "/webhook/taiga?prj=P", data=json.dumps(payload), content_type="application/json", - headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"} + headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"}, ) assert resp.status_code == 200 data = resp.get_json() @@ -74,7 +89,7 @@ def test_delete_action(self, mock_verify, mock_coll, client): "type": "task", "action": "delete", "data": {"id": 99, "project": {"name": "P"}}, - "by": {"username": "u"} + "by": {"username": "u"}, } mock_collection = MagicMock() mock_coll.return_value = mock_collection @@ -83,7 +98,7 @@ def test_delete_action(self, mock_verify, mock_coll, client): "/webhook/taiga?prj=P", data=json.dumps(payload), content_type="application/json", - headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"} + headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"}, ) assert resp.status_code == 200 mock_collection.delete_one.assert_called_once_with({"task_id": 99}) @@ -95,7 +110,7 @@ def test_delete_no_id_returns_400(self, mock_verify, mock_coll, client): "type": "task", "action": "delete", "data": {"id": "", "project": {"name": "P"}}, - "by": {"username": "u"} + "by": {"username": "u"}, } mock_coll.return_value = MagicMock() @@ -103,7 +118,7 @@ def test_delete_no_id_returns_400(self, mock_verify, mock_coll, client): "/webhook/taiga?prj=P", data=json.dumps(payload), content_type="application/json", - headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"} + headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"}, ) assert resp.status_code == 400 @@ -111,11 +126,19 @@ def test_delete_no_id_returns_400(self, mock_verify, mock_coll, client): @patch("routes.taiga_routes.get_collection") @patch("routes.taiga_routes.parse_taiga_event") @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) - def test_userstory_upsert(self, mock_verify, mock_parse, mock_coll, mock_notify, client, taiga_userstory_payload): + def test_userstory_upsert( + self, + mock_verify, + mock_parse, + mock_coll, + mock_notify, + client, + taiga_userstory_payload, + ): mock_parse.return_value = { "event_type": "userstory", "userstory_id": 400, - "assigned_by": "u" + "assigned_by": "u", } mock_collection = MagicMock() mock_coll.return_value = mock_collection @@ -128,11 +151,19 @@ def test_userstory_upsert(self, mock_verify, mock_parse, mock_coll, mock_notify, @patch("routes.taiga_routes.get_collection") @patch("routes.taiga_routes.parse_taiga_event") @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) - def test_epic_upsert(self, mock_verify, mock_parse, mock_coll, mock_notify, client, taiga_epic_payload): + def test_epic_upsert( + self, + mock_verify, + mock_parse, + mock_coll, + mock_notify, + client, + taiga_epic_payload, + ): mock_parse.return_value = { "event_type": "epic", "epic_id": 300, - "assigned_by": "u" + "assigned_by": "u", } mock_collection = MagicMock() mock_coll.return_value = mock_collection @@ -145,11 +176,19 @@ def test_epic_upsert(self, mock_verify, mock_parse, mock_coll, mock_notify, clie @patch("routes.taiga_routes.get_collection") @patch("routes.taiga_routes.parse_taiga_event") @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) - def test_issue_upsert(self, mock_verify, mock_parse, mock_coll, mock_notify, client, taiga_issue_payload): + def test_issue_upsert( + self, + mock_verify, + mock_parse, + mock_coll, + mock_notify, + client, + taiga_issue_payload, + ): mock_parse.return_value = { "event_type": "issue", "issue_id": 200, - "assigned_by": "u" + "assigned_by": "u", } mock_collection = MagicMock() mock_coll.return_value = mock_collection @@ -162,11 +201,19 @@ def test_issue_upsert(self, mock_verify, mock_parse, mock_coll, mock_notify, cli @patch("routes.taiga_routes.get_collection") @patch("routes.taiga_routes.parse_taiga_event") @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) - def test_notify_eval_error_returns_500(self, mock_verify, mock_parse, mock_coll, mock_notify, client, taiga_task_payload): + def test_notify_eval_error_returns_500( + self, + mock_verify, + mock_parse, + mock_coll, + mock_notify, + client, + taiga_task_payload, + ): mock_parse.return_value = { "event_type": "task", "task_id": 100, - "assigned_by": "u" + "assigned_by": "u", } mock_coll.return_value = MagicMock() @@ -177,17 +224,19 @@ def test_notify_eval_error_returns_500(self, mock_verify, mock_parse, mock_coll, @patch("routes.taiga_routes.get_collection") @patch("routes.taiga_routes.parse_taiga_event") @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) - def test_userstory_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, mock_notify, client): + def test_userstory_no_id_returns_400( + self, mock_verify, mock_parse, mock_coll, mock_notify, client + ): payload = { "type": "userstory", "action": "create", "by": {"username": "u"}, - "data": {"id": 1, "project": {"id": 1, "name": "P"}} + "data": {"id": 1, "project": {"id": 1, "name": "P"}}, } mock_parse.return_value = { "event_type": "userstory", "userstory_id": None, - "assigned_by": "u" + "assigned_by": "u", } mock_coll.return_value = MagicMock() @@ -195,7 +244,7 @@ def test_userstory_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, m "/webhook/taiga?prj=P", data=json.dumps(payload), content_type="application/json", - headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"} + headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"}, ) assert resp.status_code == 400 @@ -203,17 +252,19 @@ def test_userstory_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, m @patch("routes.taiga_routes.get_collection") @patch("routes.taiga_routes.parse_taiga_event") @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) - def test_task_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, mock_notify, client): + def test_task_no_id_returns_400( + self, mock_verify, mock_parse, mock_coll, mock_notify, client + ): payload = { "type": "task", "action": "create", "by": {"username": "u"}, - "data": {"id": 1, "project": {"id": 1, "name": "P"}} + "data": {"id": 1, "project": {"id": 1, "name": "P"}}, } mock_parse.return_value = { "event_type": "task", "task_id": None, - "assigned_by": "u" + "assigned_by": "u", } mock_coll.return_value = MagicMock() @@ -221,7 +272,7 @@ def test_task_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, mock_n "/webhook/taiga?prj=P", data=json.dumps(payload), content_type="application/json", - headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"} + headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"}, ) assert resp.status_code == 400 @@ -229,17 +280,19 @@ def test_task_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, mock_n @patch("routes.taiga_routes.get_collection") @patch("routes.taiga_routes.parse_taiga_event") @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) - def test_epic_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, mock_notify, client): + def test_epic_no_id_returns_400( + self, mock_verify, mock_parse, mock_coll, mock_notify, client + ): payload = { "type": "epic", "action": "create", "by": {"username": "u"}, - "data": {"id": 1, "project": {"id": 1, "name": "P"}} + "data": {"id": 1, "project": {"id": 1, "name": "P"}}, } mock_parse.return_value = { "event_type": "epic", "epic_id": None, - "assigned_by": "u" + "assigned_by": "u", } mock_coll.return_value = MagicMock() @@ -247,7 +300,7 @@ def test_epic_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, mock_n "/webhook/taiga?prj=P", data=json.dumps(payload), content_type="application/json", - headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"} + headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"}, ) assert resp.status_code == 400 @@ -255,17 +308,19 @@ def test_epic_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, mock_n @patch("routes.taiga_routes.get_collection") @patch("routes.taiga_routes.parse_taiga_event") @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) - def test_issue_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, mock_notify, client): + def test_issue_no_id_returns_400( + self, mock_verify, mock_parse, mock_coll, mock_notify, client + ): payload = { "type": "issue", "action": "create", "by": {"username": "u"}, - "data": {"id": 1, "project": {"id": 1, "name": "P"}} + "data": {"id": 1, "project": {"id": 1, "name": "P"}}, } mock_parse.return_value = { "event_type": "issue", "issue_id": None, - "assigned_by": "u" + "assigned_by": "u", } mock_coll.return_value = MagicMock() @@ -273,7 +328,7 @@ def test_issue_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, mock_ "/webhook/taiga?prj=P", data=json.dumps(payload), content_type="application/json", - headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"} + headers={"X-TAIGA-WEBHOOK-SIGNATURE": "x"}, ) assert resp.status_code == 400 @@ -281,8 +336,20 @@ def test_issue_no_id_returns_400(self, mock_verify, mock_parse, mock_coll, mock_ @patch("routes.taiga_routes.get_collection") @patch("routes.taiga_routes.parse_taiga_event") @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) - def test_collection_name_task(self, mock_verify, mock_parse, mock_coll, mock_notify, client, taiga_task_payload): - mock_parse.return_value = {"event_type": "task", "task_id": 1, "assigned_by": "u"} + def test_collection_name_task( + self, + mock_verify, + mock_parse, + mock_coll, + mock_notify, + client, + taiga_task_payload, + ): + mock_parse.return_value = { + "event_type": "task", + "task_id": 1, + "assigned_by": "u", + } mock_coll.return_value = MagicMock() self._post(client, taiga_task_payload, prj="MyPrj") @@ -292,8 +359,20 @@ def test_collection_name_task(self, mock_verify, mock_parse, mock_coll, mock_not @patch("routes.taiga_routes.get_collection") @patch("routes.taiga_routes.parse_taiga_event") @patch("routes.taiga_routes.verify_taiga_signature", return_value=True) - def test_collection_name_userstory(self, mock_verify, mock_parse, mock_coll, mock_notify, client, taiga_userstory_payload): - mock_parse.return_value = {"event_type": "userstory", "userstory_id": 1, "assigned_by": "u"} + def test_collection_name_userstory( + self, + mock_verify, + mock_parse, + mock_coll, + mock_notify, + client, + taiga_userstory_payload, + ): + mock_parse.return_value = { + "event_type": "userstory", + "userstory_id": 1, + "assigned_by": "u", + } mock_coll.return_value = MagicMock() self._post(client, taiga_userstory_payload, prj="MyPrj") diff --git a/tests/test_verify_signature_github.py b/tests/test_verify_signature_github.py index 75ce5ac..04f6cca 100644 --- a/tests/test_verify_signature_github.py +++ b/tests/test_verify_signature_github.py @@ -1,5 +1,7 @@ """Tests for routes/verify_signature/verify_signature_github.py""" -import hashlib, hmac, pytest + +import hashlib +import hmac from unittest.mock import MagicMock from routes.verify_signature.verify_signature_github import verify_github_signature diff --git a/tests/test_verify_signature_taiga.py b/tests/test_verify_signature_taiga.py index dbd1cb6..a48bb81 100644 --- a/tests/test_verify_signature_taiga.py +++ b/tests/test_verify_signature_taiga.py @@ -1,5 +1,7 @@ """Tests for routes/verify_signature/verify_signature_taiga.py""" -import hashlib, hmac, pytest + +import hashlib +import hmac from unittest.mock import MagicMock from routes.verify_signature.verify_signature_taiga import verify_taiga_signature diff --git a/utils/datetime_utils.py b/utils/datetime_utils.py index db536c7..7edd72b 100644 --- a/utils/datetime_utils.py +++ b/utils/datetime_utils.py @@ -6,7 +6,7 @@ def to_madrid_local(ts: str) -> str: """ Receive date in ISO-8601 then transforms it to Europe/Madrid date. """ - if not ts: # '', None… + if not ts: # '', None… return ts # The date standard only accepts '+00:00', but taiga returns in 'Z' format dt_utc = datetime.fromisoformat(ts.replace("Z", "+00:00")) diff --git a/utils/recovery/github_recovery.py b/utils/recovery/github_recovery.py index ad29985..1f2d8d4 100644 --- a/utils/recovery/github_recovery.py +++ b/utils/recovery/github_recovery.py @@ -1,4 +1,5 @@ -import argparse, requests +import argparse +import requests from typing import Dict, Iterable, Optional, List from datetime import datetime, timezone from pymongo import UpdateOne @@ -14,10 +15,11 @@ setup_logging() logger = logging.getLogger(__name__) + def parse_dt(s: str | None) -> Optional[datetime]: - ''' + """ Accepts ‘2025-05-01’, ‘2025-05-01T14:30’, etc. And returns the datetime tz-aware (Madrid). - ''' + """ if not s: return None d = dtp.isoparse(s) @@ -35,9 +37,9 @@ def get_organization_repos(org: str, headers: Dict[str, str]) -> List[str]: def gh_paginated(url: str, headers: Dict[str, str]) -> Iterable[Dict]: - ''' + """ Gets paginated results from a GitHub API endpoint. With this each call to the API returns a suitable JSON - ''' + """ while url: r = requests.get(url, headers=headers, timeout=30) r.raise_for_status() @@ -46,85 +48,124 @@ def gh_paginated(url: str, headers: Dict[str, str]) -> Iterable[Dict]: def upsert(coll, docs: list[dict], key: str) -> int: - ''' - Upserts a list of documents into a MongoDB collection. - ''' + """ + Upserts a list of documents into a MongoDB collection. + """ if not docs: return 0 - operations = [UpdateOne({key: d[key]}, {"$set": d}, upsert=True) for d in docs] #Create a list of UpdateOne operations for each document in the list, using the key to identify the document + operations = [ + UpdateOne({key: d[key]}, {"$set": d}, upsert=True) for d in docs + ] # Create a list of UpdateOne operations for each document in the list, using the key to identify the document res = coll.bulk_write(operations, ordered=False) return res.matched_count + len(res.upserted_ids) - - -def collect_github(org: str, repo: str, prj: str, events: list[str], since: Optional[str], until: Optional[str], quality_model: str): - ''' +def collect_github( + org: str, + repo: str, + prj: str, + events: list[str], + since: Optional[str], + until: Optional[str], + quality_model: str, +): + """ Collects data from a GitHub repository and inserts it into MongoDB. Follows the schema of the LD-Connect project. - ''' - repo_full = f"{org}/{repo}" # Full name of the repository, e.g. "LD-Connect/ld-connect" + """ + repo_full = ( + f"{org}/{repo}" # Full name of the repository, e.g. "LD-Connect/ld-connect" + ) headers = {"Accept": "application/vnd.github+json"} if GITHUB_TOKEN: - headers["Authorization"] = f"Bearer {GITHUB_TOKEN}" # Authentication with GitHub API using a token - - counters = {"commits": 0, "issues": 0, "pull_requests": 0} #Counter to display the number of documents inserted of each event type - author_login = "backfill" # The author login is always "backfill" for backfilling - - for ev in events: # Start iterating over the events to collect - - if ev == "commits": #First commits - event_name= "push" # The event name for commits is always "push" - + headers["Authorization"] = ( + f"Bearer {GITHUB_TOKEN}" # Authentication with GitHub API using a token + ) + + counters = { + "commits": 0, + "issues": 0, + "pull_requests": 0, + } # Counter to display the number of documents inserted of each event type + author_login = "backfill" # The author login is always "backfill" for backfilling + + for ev in events: # Start iterating over the events to collect + + if ev == "commits": # First commits + event_name = "push" # The event name for commits is always "push" + log_url = f"https://api.github.com/repos/{repo_full}/commits?per_page=100" - if since: log_url += f"&since={since}" # If a SINCE date is proviaded, add it to the URL - if until: log_url += f"&until={until}" # If a UNTIL date is proviaded, add it to the URL + if since: + log_url += ( + f"&since={since}" # If a SINCE date is proviaded, add it to the URL + ) + if until: + log_url += ( + f"&until={until}" # If a UNTIL date is proviaded, add it to the URL + ) - payloads = [] #List to store the payloads of the commits - for c in gh_paginated(log_url, headers): # Iterate over the paginated results of the commits and store them in the payloads list under the schema - payloads.append({ - "X-GitHub-Event": "push", - "repository": {"full_name": repo_full}, - "organization": {"login": org}, - "sender": c["author"] or {}, - "commits": [{ - "id": c["sha"], - "url": c["url"], - "message": c["commit"]["message"], - "timestamp": c["commit"]["author"]["date"], - "author": { - "username": (c["author"] or {}).get("login", ""), - "name": c["commit"]["author"]["name"], - "email": c["commit"]["author"]["email"], - }, - }], - }) - - coll = get_collection(f"github_{prj}.commits") # Collection name to store - for raw in payloads: # All the raw payloads, parse them and insert in collection - doc = parse_github_event(raw,prj) + payloads = [] # List to store the payloads of the commits + for c in gh_paginated( + log_url, headers + ): # Iterate over the paginated results of the commits and store them in the payloads list under the schema + payloads.append( + { + "X-GitHub-Event": "push", + "repository": {"full_name": repo_full}, + "organization": {"login": org}, + "sender": c["author"] or {}, + "commits": [ + { + "id": c["sha"], + "url": c["url"], + "message": c["commit"]["message"], + "timestamp": c["commit"]["author"]["date"], + "author": { + "username": (c["author"] or {}).get("login", ""), + "name": c["commit"]["author"]["name"], + "email": c["commit"]["author"]["email"], + }, + } + ], + } + ) + + coll = get_collection(f"github_{prj}.commits") # Collection name to store + for ( + raw + ) in payloads: # All the raw payloads, parse them and insert in collection + doc = parse_github_event(raw, prj) for c in doc["commits"]: - c.update({"team_name": doc["team_name"], - "repo_name": doc["repo_name"], - "sender_info": doc["sender_info"], - "prj": prj}) + c.update( + { + "team_name": doc["team_name"], + "repo_name": doc["repo_name"], + "sender_info": doc["sender_info"], + "prj": prj, + } + ) counters["commits"] += upsert(coll, [c], "sha") - #COMMUNICATION WITH LD_EVAL USING API - logger.info(f"Notifying LD_EVAL about event: {event_name} for team with external_id: {prj} with quality_model: {quality_model}") + # COMMUNICATION WITH LD_EVAL USING API + logger.info( + f"Notifying LD_EVAL about event: {event_name} for team with external_id: {prj} with quality_model: {quality_model}" + ) try: notify_eval_push(event_name, prj, author_login, quality_model) except Exception as e: logger.error(f"Error notifying LD_EVAL: {e}") - - - - elif ev == "issues": # Issue event - event_name = "issues" # The event name for issues is always "issues" + + elif ev == "issues": # Issue event + event_name = "issues" # The event name for issues is always "issues" url = f"https://api.github.com/repos/{repo_full}/issues?state=all&per_page=100" - if since: url += f"&since={since}" + if since: + url += f"&since={since}" raw_issues = list(gh_paginated(url, headers)) - coll = get_collection(f"github_{prj}.issues") #Collection name to store - for i in raw_issues: # All the raw payloads, parse them and insert in collection + coll = get_collection(f"github_{prj}.issues") # Collection name to store + for ( + i + ) in ( + raw_issues + ): # All the raw payloads, parse them and insert in collection payload = { "X-GitHub-Event": "issues", "action": i["state"], @@ -137,19 +178,22 @@ def collect_github(org: str, repo: str, prj: str, events: list[str], since: Opt doc["prj"] = prj doc["issue_id"] = doc["issue"]["number"] counters["issues"] += upsert(coll, [doc], "issue_id") - #COMMUNICATION WITH LD_EVAL USING API - logger.info(f"Notifying LD_EVAL about event: {event_name} for team with external_id: {prj} with quality_model: {quality_model}") + # COMMUNICATION WITH LD_EVAL USING API + logger.info( + f"Notifying LD_EVAL about event: {event_name} for team with external_id: {prj} with quality_model: {quality_model}" + ) try: notify_eval_push(event_name, prj, author_login, quality_model) except Exception as e: logger.error(f"Error notifying LD_EVAL: {e}") - - - elif ev == "pull_requests": #Pull request event - event_name = "pull_requests" # The event name for pull requests is always "pull_requests" + + elif ev == "pull_requests": # Pull request event + event_name = "pull_requests" # The event name for pull requests is always "pull_requests" url = f"https://api.github.com/repos/{repo_full}/pulls?state=closed&per_page=100" raw_prs = list(gh_paginated(url, headers)) - coll = get_collection(f"github_{prj}.pull_requests") # All the raw payloads, parse them and insert in collection + coll = get_collection( + f"github_{prj}.pull_requests" + ) # All the raw payloads, parse them and insert in collection for p in raw_prs: payload = { "X-GitHub-Event": "pull_request", @@ -162,56 +206,105 @@ def collect_github(org: str, repo: str, prj: str, events: list[str], since: Opt doc = parse_github_event(payload, prj) doc["prj"] = prj counters["pull_requests"] += upsert(coll, [doc], "pr_number") - #COMMUNICATION WITH LD_EVAL USING API - logger.info(f"Notifying LD_EVAL about event: {event_name} for team with external_id: {prj} with quality_model: {quality_model}") + # COMMUNICATION WITH LD_EVAL USING API + logger.info( + f"Notifying LD_EVAL about event: {event_name} for team with external_id: {prj} with quality_model: {quality_model}" + ) try: notify_eval_push(event_name, prj, author_login, quality_model) except Exception as e: logger.error(f"Error notifying LD_EVAL: {e}") - - + else: logger.warning("Event %s not supported.", ev) - for k in ("commits", "issues", "pull_requests"): # Log the counters of each event type + for k in ( + "commits", + "issues", + "pull_requests", + ): # Log the counters of each event type if k in events: - logger.info(" • %s → %d documents", k.replace('_', ' '), counters[k]) + logger.info(" • %s → %d documents", k.replace("_", " "), counters[k]) logger.info("%d documents inserted.", sum(counters.values())) - - if __name__ == "__main__": - ap = argparse.ArgumentParser(description="Back-fill of GITHUB for a project, to insert it in MongoDB") - ap.add_argument("--org", required=True, help="Organization name on Github") - ap.add_argument("--repo", help="Repository name on Github, if not provided, it will backfill all repositories in the organization") - ap.add_argument("--prj", required=True, help="External ID of the project in LD-Connect") - ap.add_argument("--events", default="commits,issues,pull_requests",help="List separated by commas pf the events to backfill, by default: commits,issues,pull_requests") - ap.add_argument("--from-date", help="Date FROM the backfill will be made, must be in format (YYYY-MM-DD), can also put a full date with time (2025-05-01T14:30)") - ap.add_argument("--to-date", help="Date UNTIL the backfill will be made, must be in format (YYYY-MM-DD), can also put a full date with time (2025-05-01T14:30)") - ap.add_argument("--quality-model", default="default", help="Sets the quality model to use for the evaluation, by default: default") + ap = argparse.ArgumentParser( + description="Back-fill of GITHUB for a project, to insert it in MongoDB" + ) + ap.add_argument("--org", required=True, help="Organization name on Github") + ap.add_argument( + "--repo", + help="Repository name on Github, if not provided, it will backfill all repositories in the organization", + ) + ap.add_argument( + "--prj", required=True, help="External ID of the project in LD-Connect" + ) + ap.add_argument( + "--events", + default="commits,issues,pull_requests", + help="List separated by commas pf the events to backfill, by default: commits,issues,pull_requests", + ) + ap.add_argument( + "--from-date", + help="Date FROM the backfill will be made, must be in format (YYYY-MM-DD), can also put a full date with time (2025-05-01T14:30)", + ) + ap.add_argument( + "--to-date", + help="Date UNTIL the backfill will be made, must be in format (YYYY-MM-DD), can also put a full date with time (2025-05-01T14:30)", + ) + ap.add_argument( + "--quality-model", + default="default", + help="Sets the quality model to use for the evaluation, by default: default", + ) ns = ap.parse_args() events = [e.strip() for e in ns.events.split(",") if e.strip()] - since = parse_dt(ns.from_date).astimezone(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") if ns.from_date else None - until = parse_dt(ns.to_date).astimezone(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") if ns.to_date else None + since = ( + parse_dt(ns.from_date) + .astimezone(timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z") + if ns.from_date + else None + ) + until = ( + parse_dt(ns.to_date) + .astimezone(timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z") + if ns.to_date + else None + ) headers = {"Accept": "application/vnd.github+json"} if GITHUB_TOKEN: - headers["Authorization"] = f"Bearer {GITHUB_TOKEN}" # Authentication with GitHub API using a token + headers["Authorization"] = ( + f"Bearer {GITHUB_TOKEN}" # Authentication with GitHub API using a token + ) if ns.repo: - repos= [ns.repo] + repos = [ns.repo] else: repos = get_organization_repos(ns.org, headers) - + if not repos: raise SystemExit(f"No repositories found for organization {ns.org}.") - - + for repo in repos: - collect_github( org = ns.org, repo = repo, prj= ns.prj, events= events, since = since, until= until, quality_model=ns.quality_model) + collect_github( + org=ns.org, + repo=repo, + prj=ns.prj, + events=events, + since=since, + until=until, + quality_model=ns.quality_model, + ) # In order to execute this script: python -m recovery.github_recovery --org LD-Connect --repo ld-connect --prj LD_Test_Project --events commits,issues,pull_requests --from-date 2025-01-01 --to-date 2025-12-31 -# Or python -m recovery.github_recovery --org LD-Connect --repo ld-connect --prj LD_Test_Project \ No newline at end of file +# Or python -m recovery.github_recovery --org LD-Connect --repo ld-connect --prj LD_Test_Project diff --git a/utils/recovery/taiga_recovery.py b/utils/recovery/taiga_recovery.py index ff34dcb..b63cb25 100644 --- a/utils/recovery/taiga_recovery.py +++ b/utils/recovery/taiga_recovery.py @@ -1,22 +1,22 @@ -import argparse, re, requests +import argparse +import re +import requests from pymongo import UpdateOne from datetime import datetime, timezone from typing import Optional, Dict, List -from dateutil import tz, parser as dtparser # pip install python-dateutil +from dateutil import tz, parser as dtparser # pip install python-dateutil import logging from database.mongo_client import get_collection -from utils.taiga_token.get_taiga_token import get_token from routes.API_publisher.API_event_publisher import notify_eval_push from config.logger_config import setup_logging -from config.settings import TAIGA_USERNAME, TAIGA_PASSWORD, TAIGA_API_URL +from config.settings import TAIGA_API_URL setup_logging() logger = logging.getLogger(__name__) - def parse_dt(s: str) -> datetime: """Accepts ‘2025-05-01’, ‘2025-05-01T14:30’, etc. And returns the datetime tz-aware (Madrid).""" d = dtparser.isoparse(s) @@ -27,10 +27,10 @@ def parse_dt(s: str) -> datetime: # Functions to interact with Taiga API def get_username_id(token: str) -> int: - ''' + """ Given a Taiga API token, return the user ID of the authenticated user. With this ID we canfind the projects that the user is a member of. - ''' + """ h = {"Authorization": f"Bearer {token}"} r = requests.get(f"{TAIGA_API_URL}/users/me", headers=h, timeout=10) r.raise_for_status() @@ -44,20 +44,28 @@ def get_project_id_by_slug(slug: str) -> int: if r.status_code == 200: return r.json()["id"] if r.status_code in (401, 403): - raise SystemExit(f"Project ‘{slug}’ is private. Provide credentials or make it public.") + raise SystemExit( + f"Project ‘{slug}’ is private. Provide credentials or make it public." + ) if r.status_code == 404: raise SystemExit(f"Project slug ‘{slug}’ not found.") r.raise_for_status() assert False # pragma: no cover - + + def get_project_id_by_username_id(project_name: str, token: str) -> int: - ''' + """ Given a project name and a Taiga API token, return the project ID. With the projecta ID we can fetch the entities (tasks, issues, etc.) of the project. - ''' + """ h = {"Authorization": f"Bearer {token}"} uid = get_username_id(token) - r = requests.get(f"{TAIGA_API_URL}/projects", headers=h, params={"member": uid}, timeout=10) + r = requests.get( + f"{TAIGA_API_URL}/projects", + headers=h, + params={"member": uid}, + timeout=10, + ) r.raise_for_status() for p in r.json(): if p["name"].lower() == project_name.lower(): @@ -65,129 +73,152 @@ def get_project_id_by_username_id(project_name: str, token: str) -> int: raise SystemExit(f"Project named «{project_name}» is not under the introduced user") -def fetch_entities(entity: str, project: int, start: Optional[datetime] = None, end: Optional[datetime] = None) -> List[Dict]: - ''' +def fetch_entities( + entity: str, + project: int, + start: Optional[datetime] = None, + end: Optional[datetime] = None, +) -> List[Dict]: + """ Given an entity type (tasks, issues, epics, userstories), a project ID, and a token, the function fetches the entities from the Taiga API. The start and end parameters are optional and can be used to filter the entities by creation date. - - ''' + + """ if entity not in ENTITY_ENDPOINT: raise ValueError(f"Not supported type: {entity}") - endpoint_path = ENTITY_ENDPOINT[entity][0] + endpoint_path = ENTITY_ENDPOINT[entity][0] headers = { "x-disable-pagination": "True", } params = {"project": project} - if start: params["modified_date__gte"] = start.astimezone(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") # See if there is a start date to fetch data, if there is add it to the params - if end: params["modified_date__lte"] = end.astimezone(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") # See if there is a end date to fetch data, if there is add it to the params - - r = requests.get(f"{TAIGA_API_URL}/{endpoint_path}", headers=headers, params=params, timeout=30) # In the request we add the headers and params + if start: + params["modified_date__gte"] = ( + start.astimezone(timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z") + ) # See if there is a start date to fetch data, if there is add it to the params + if end: + params["modified_date__lte"] = ( + end.astimezone(timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z") + ) # See if there is a end date to fetch data, if there is add it to the params + + r = requests.get( + f"https://api.taiga.io/api/v1/{endpoint_path}", + headers=headers, + params=params, + timeout=30, + ) # In the request we add the headers and params r.raise_for_status() return r.json() - def upsert(coll, docs: list[dict], key: str) -> int: - ''' - Upserts a list of documents into a MongoDB collection. - ''' + """ + Upserts a list of documents into a MongoDB collection. + """ if not docs: return 0 - operations = [UpdateOne({key: d[key]}, {"$set": d}, upsert=True) for d in docs] #Create a list of UpdateOne operations for each document in the list, using the key to identify the document + operations = [ + UpdateOne({key: d[key]}, {"$set": d}, upsert=True) for d in docs + ] # Create a list of UpdateOne operations for each document in the list, using the key to identify the document res = coll.bulk_write(operations, ordered=False) return res.matched_count + len(res.upserted_ids) # Converters from API Schema to MongoDB Schema def task_from_api(j: dict, prj: str) -> dict: - ''' + """ Converts a task JSON object from the Taiga API to a MongoDB document schema. - ''' + """ m = j.get("milestone_extra_info") or {} us = j.get("user_story_extra_info") or {} return { - "task_id": j["id"], - "action_type": "import", - "assigned_by": "backfill", - "assigned_to": (j.get("assigned_to_extra_info") or {}).get("username"), - "created_date": j["created_date"], + "task_id": j["id"], + "action_type": "import", + "assigned_by": "backfill", + "assigned_to": (j.get("assigned_to_extra_info") or {}).get("username"), + "created_date": j["created_date"], "custom_attributes": j.get("custom_attributes_values") or {}, "estimated_finish": m.get("estimated_finish"), - "estimated_start": m.get("estimated_start"), - "event_type": "task", - "finished_date": j["finished_date"], - "is_closed": j["status_extra_info"]["is_closed"], + "estimated_start": m.get("estimated_start"), + "event_type": "task", + "finished_date": j["finished_date"], + "is_closed": j["status_extra_info"]["is_closed"], "milestone_closed": m.get("closed"), "milestone_created_date": m.get("created_date"), - "milestone_id": j.get("milestone"), + "milestone_id": j.get("milestone"), "milestone_modified_date": m.get("modified_date"), "milestone_name": m.get("name"), - "modified_date": j["modified_date"], - "prj": prj, - "reference": j["ref"], - "status": j["status_extra_info"]["name"], - "subject": j["subject"], - "team_name": j["project_extra_info"]["name"], - "userstory_id": j.get("user_story"), - "userstory_is_closed": us.get("is_closed"), + "modified_date": j["modified_date"], + "prj": prj, + "reference": j["ref"], + "status": j["status_extra_info"]["name"], + "subject": j["subject"], + "team_name": j["project_extra_info"]["name"], + "userstory_id": j.get("user_story"), + "userstory_is_closed": us.get("is_closed"), } def issue_from_api(j: dict, prj: str) -> dict: - ''' + """ Converts a isue JSON object from the Taiga API to a MongoDB document schema. - ''' + """ return { - "issue_id": j["id"], + "issue_id": j["id"], "action_type": "import", - "assigned_by": "backfill", + "assigned_by": "backfill", "assigned_to": (j.get("assigned_to_extra_info") or {}).get("username"), - "created_date": j["created_date"], + "created_date": j["created_date"], "description": j.get("description"), - "due_date": j.get("due_date"), - "event_type": "issue", + "due_date": j.get("due_date"), + "event_type": "issue", "finished_date": j.get("finished_date"), - "is_closed": (j.get("status_extra_info") or {}).get("is_closed"), + "is_closed": (j.get("status_extra_info") or {}).get("is_closed"), "modified_date": j["modified_date"], - "priority": (j.get("priority_extra_info") or {}).get("name"), - "prj": prj, - "severity": (j.get("severity_extra_info") or {}).get("name"), - "status": (j.get("status_extra_info") or {}).get("name"), - "subject": j["subject"], - "team_name": j["project_extra_info"]["name"], - "type": (j.get("type_extra_info") or {}).get("name"), + "priority": (j.get("priority_extra_info") or {}).get("name"), + "prj": prj, + "severity": (j.get("severity_extra_info") or {}).get("name"), + "status": (j.get("status_extra_info") or {}).get("name"), + "subject": j["subject"], + "team_name": j["project_extra_info"]["name"], + "type": (j.get("type_extra_info") or {}).get("name"), } def epic_from_api(j: dict, prj: str) -> dict: - ''' + """ Converts an epic JSON object from the Taiga API to a MongoDB document schema. - ''' + """ return { - "epic_id": j["id"], - "action_type": "import", - "assigned_by": "backfill", - "created_date": j["created_date"], - "event_type": "epic", - "is_closed": (j.get("status_extra_info") or {}).get("is_closed"), + "epic_id": j["id"], + "action_type": "import", + "assigned_by": "backfill", + "created_date": j["created_date"], + "event_type": "epic", + "is_closed": (j.get("status_extra_info") or {}).get("is_closed"), "modified_date": j["modified_date"], - "prj": prj, - "project_id": j["project_extra_info"]["id"], - "status": (j.get("status_extra_info") or {}).get("name"), - "subject": j["subject"], - "team_name": j["project_extra_info"]["name"], + "prj": prj, + "project_id": j["project_extra_info"]["id"], + "status": (j.get("status_extra_info") or {}).get("name"), + "subject": j["subject"], + "team_name": j["project_extra_info"]["name"], } - - + + def userstory_from_api(j: dict, prj: str) -> dict: - ''' + """ Converts an userstory JSON object from the Taiga API to a MongoDB document schema. - ''' + """ m = j.get("milestone_extra_info") or {} desc = j.get("description") or "" pattern = bool(re.search(r"as\s+.*?\s+i want\s+.*?\s+so that\s+.*", desc, re.I)) - raw_points = j.get("points") # puede ser list | "" | None + raw_points = j.get("points") # puede ser list | "" | None if isinstance(raw_points, list): total = sum((p.get("value") or 0) for p in raw_points) else: @@ -196,58 +227,78 @@ def userstory_from_api(j: dict, prj: str) -> dict: return { "userstory_id": j["id"], "action_type": "import", - "assigned_by": "backfill", - "created_date": j["created_date"], + "assigned_by": "backfill", + "created_date": j["created_date"], "custom_attributes": j.get("custom_attributes_values") or {}, "estimated_finish": m.get("estimated_finish"), - "estimated_start": m.get("estimated_start"), - "event_type": "userstory", - "is_closed": (j.get("status_extra_info") or {}).get("is_closed"), + "estimated_start": m.get("estimated_start"), + "event_type": "userstory", + "is_closed": (j.get("status_extra_info") or {}).get("is_closed"), "milestone_closed": m.get("closed"), "milestone_created_date": m.get("created_date"), - "milestone_id": j.get("milestone"), + "milestone_id": j.get("milestone"), "milestone_modified_date": m.get("modified_date"), "milestone_name": m.get("name"), "modified_date": j["modified_date"], "pattern": pattern, - "priority": (j.get("custom_attributes_values") or {}).get("Priority"), - "prj": prj, - "status": (j.get("status_extra_info") or {}).get("name"), - "subject": j["subject"], - "team_name": j["project_extra_info"]["name"], - "total_points": total, + "priority": (j.get("custom_attributes_values") or {}).get("Priority"), + "prj": prj, + "status": (j.get("status_extra_info") or {}).get("name"), + "subject": j["subject"], + "team_name": j["project_extra_info"]["name"], + "total_points": total, } - - -ENTITY_ENDPOINT = { - "task": ("tasks", task_from_api, "task_id"), - "issue": ("issues", issue_from_api, "issue_id"), - "epic": ("epics", epic_from_api, "epic_id"), - "userstory": ("userstories", userstory_from_api, "userstory_id"), - } - +ENTITY_ENDPOINT = { + "task": ("tasks", task_from_api, "task_id"), + "issue": ("issues", issue_from_api, "issue_id"), + "epic": ("epics", epic_from_api, "epic_id"), + "userstory": ("userstories", userstory_from_api, "userstory_id"), +} def main(argv: list[str] | None = None): - ap = argparse.ArgumentParser(description="Back-fill of Taiga for a project, to insert it in MongoDB") - #ap.add_argument("--project", required=True, help="Name of the project in Taiga") - ap.add_argument("--slug", "--project", dest="slug", required=True,help="Slug públic del projecte a Taiga") - #ap.add_argument("--project", required=True, help="Name of the project in Taiga") - ap.add_argument("--prj", required=True, help="External ID of the project in LD-Connect") + ap = argparse.ArgumentParser( + description="Back-fill of Taiga for a project, to insert it in MongoDB" + ) + # ap.add_argument("--project", required=True, help="Name of the project in Taiga") + ap.add_argument( + "--slug", + "--project", + dest="slug", + required=True, + help="Slug públic del projecte a Taiga", + ) + # ap.add_argument("--project", required=True, help="Name of the project in Taiga") + ap.add_argument( + "--prj", required=True, help="External ID of the project in LD-Connect" + ) # ap.add_argument("--taiga-user", required=True, help="Username in Taiga of the teacher with acces to all Students Projects") # ap.add_argument("--taiga-pass", required=True, help="Password in Taiga of the teacher with acces to all Students Projects") - ap.add_argument("--events", default="task,userstory,issue,epic",help="List separated by commas pf the events to backfill, by default: task,userstory,issue,epic") - ap.add_argument("--from-date", help="Date FROM the backfill will be made, must be in format (2025-05-01), can also put a full date with time (2025-05-01T14:30)") - ap.add_argument("--to-date", help="Date UNTIL the backfill will be made, must be in format (2025-05-01), can also put a full date with time (2025-05-01T14:30)") - ap.add_argument("--quality-model", default="default", help="Sets the quality model to use for the evaluation, by default: default") - - - ns = ap.parse_args(argv) - events = [e.strip().lower() for e in ns.events.split(",") if e.strip()] - start = parse_dt(ns.from_date) if ns.from_date else None - end = parse_dt(ns.to_date) if ns.to_date else None + ap.add_argument( + "--events", + default="task,userstory,issue,epic", + help="List separated by commas pf the events to backfill, by default: task,userstory,issue,epic", + ) + ap.add_argument( + "--from-date", + help="Date FROM the backfill will be made, must be in format (2025-05-01), can also put a full date with time (2025-05-01T14:30)", + ) + ap.add_argument( + "--to-date", + help="Date UNTIL the backfill will be made, must be in format (2025-05-01), can also put a full date with time (2025-05-01T14:30)", + ) + ap.add_argument( + "--quality-model", + default="default", + help="Sets the quality model to use for the evaluation, by default: default", + ) + + ns = ap.parse_args(argv) + events = [e.strip().lower() for e in ns.events.split(",") if e.strip()] + start = parse_dt(ns.from_date) if ns.from_date else None + end = parse_dt(ns.to_date) if ns.to_date else None # Payload with the credentials to get the token # payload = { @@ -255,39 +306,47 @@ def main(argv: list[str] | None = None): # "password": TAIGA_PASSWORD, # "type": "normal" # } - + # token = get_token(payload) #Get the token using the credentials provided in the payload # print(f"Using token: {token}") # Print the token to the console, this is for debugging purposes - pid = get_project_id_by_slug(ns.slug) # Get the project ID using the project name and the token info - + pid = get_project_id_by_slug( + ns.slug + ) # Get the project ID using the project name and the token info - total = 0 - for event in events: # Iterate over the events to backfill + total = 0 + for event in events: # Iterate over the events to backfill endpoint, converter, key = ENTITY_ENDPOINT[event] - raw = fetch_entities(event, pid, start, end) # Get the raw data from the Taiga API for the event - docs = [converter(r, ns.prj) for r in raw] # Convert the raw data to the MongoDB schema using the converter function - coll = get_collection(f"taiga_{ns.prj}.{event}") # Get the MongoDB collection for the event - n = upsert(coll, docs, key) # Upsert the documents + raw = fetch_entities( + event, pid, start, end + ) # Get the raw data from the Taiga API for the event + docs = [ + converter(r, ns.prj) for r in raw + ] # Convert the raw data to the MongoDB schema using the converter function + coll = get_collection( + f"taiga_{ns.prj}.{event}" + ) # Get the MongoDB collection for the event + n = upsert(coll, docs, key) # Upsert the documents total += n - print(f" • {event:<12} → {n:>4} documents") # Print total number of documments - - #COMMUNICATION WITH LD_EVAL USING API - logger.info(f"Notifying LD_EVAL about event: {event} for team with external_id: {ns.prj} with quality_model: {ns.quality_model}") + logger.info(" • %s → %d documents", event, n) + + # COMMUNICATION WITH LD_EVAL USING API + logger.info( + f"Notifying LD_EVAL about event: {event} for team with external_id: {ns.prj} with quality_model: {ns.quality_model}" + ) try: notify_eval_push(event, ns.prj, "backfill", ns.quality_model) except Exception as e: logger.error(f"Error notifying LD_EVAL: {e}") - - - - span = "all time" if not (start or end) else \ - f"from {ns.from_date or '…'} to {ns.to_date or '…'}" - print(f"{total} documents inserted({span})") - + span = ( + "all time" + if not (start or end) + else f"from {ns.from_date or '…'} to {ns.to_date or '…'}" + ) + logger.info("%d documents inserted (%s)", total, span) if __name__ == "__main__": main() - -# In order to execute this script: python -m recovery.taiga_recovery --project LD_Test_Project --prj LD_Test_Project \ No newline at end of file + +# In order to execute this script: python -m recovery.taiga_recovery --project LD_Test_Project --prj LD_Test_Project diff --git a/utils/taiga_token/get_taiga_token.py b/utils/taiga_token/get_taiga_token.py index 43a36be..4e82f88 100644 --- a/utils/taiga_token/get_taiga_token.py +++ b/utils/taiga_token/get_taiga_token.py @@ -2,32 +2,30 @@ import requests logger = logging.getLogger(__name__) +REQUEST_TIMEOUT = (2, 10) -def get_token(payload:dict) -> str: +def get_token(payload: dict) -> str: """ Using a POST request of the API, this function retrieves the authentication token from Taiga. The payload must contain the username and password of the user. - + """ # Define the login endpoint URL and payload login_url = "https://api.taiga.io/api/v1/auth" - # Send the POST request to log in - response = requests.post(login_url, json=payload) + response = requests.post(login_url, json=payload, timeout=REQUEST_TIMEOUT) response.raise_for_status() # Will raise an error if the response status is not 200 # Parse the JSON response data = response.json() - # Extract the token; + # Extract the token; token = data.get("auth_token") if token: logger.info("Login successful, token retrieved.") else: logger.error("Login failed or token not found in the response.") - - return token - + return token diff --git a/utils/taiga_token/taiga_auth.py b/utils/taiga_token/taiga_auth.py index 8f9b099..c75ac1d 100644 --- a/utils/taiga_token/taiga_auth.py +++ b/utils/taiga_token/taiga_auth.py @@ -1,29 +1,29 @@ -import requests, logging, time +import requests +import logging +import time log = logging.getLogger(__name__) -_TOKENS = {} # key = (username, password) -> token +_TOKENS = {} # key = (username, password) -> token -def get_taiga_token(username:str, password: str) -> str: - ''' + +def get_taiga_token(username: str, password: str) -> str: + """ Tool to get a Taiga API token and cache it for 23 hours (In taiga documentation, says the token expires in 24h). If the token is about to expire (less than 60 seconds left), it will request a new one. This avoids making too many requests to the Taiga API for the token. - ''' - - + """ + key = (username, password) token, exp = _TOKENS.get(key, (None, 0)) - + # If the token is not set or is about to expire, request a new one if token is None or exp - time.time() < 60: - payload = { - "username": username, - "password": password, - "type": "normal" - } - r = requests.post("https://api.taiga.io/api/v1/auth", json=payload, timeout=(2, 5)) + payload = {"username": username, "password": password, "type": "normal"} + r = requests.post( + "https://api.taiga.io/api/v1/auth", json=payload, timeout=(2, 5) + ) r.raise_for_status() - + token = r.json()["auth_token"] exp = time.time() + 23 * 3600 _TOKENS[key] = (token, exp) diff --git a/utils/webhook_deletion/central_webhook_deletion.py b/utils/webhook_deletion/central_webhook_deletion.py index 8dc1607..292ff05 100644 --- a/utils/webhook_deletion/central_webhook_deletion.py +++ b/utils/webhook_deletion/central_webhook_deletion.py @@ -4,6 +4,7 @@ from config.settings import TAIGA_USERNAME, TAIGA_PASSWORD from dotenv import load_dotenv import os + # Load environment variables from the .env file load_dotenv() @@ -14,16 +15,16 @@ payload = { "username": TAIGA_USERNAME, "password": TAIGA_PASSWORD, - "type": "normal", + "type": "normal", } if __name__ == "__main__": # Get the token from Taiga token = get_token(payload) - + # Delete webhooks from GitHub delete_all_github_webhooks(WEBHOOK_URL_GITHUB) - + # Delete webhooks from Taiga - delete_all_taiga_webhooks(token, WEBHOOK_URL_TAIGA) \ No newline at end of file + delete_all_taiga_webhooks(token, WEBHOOK_URL_TAIGA) diff --git a/utils/webhook_deletion/delete_webhooks_github.py b/utils/webhook_deletion/delete_webhooks_github.py index 609923a..24ad7f2 100644 --- a/utils/webhook_deletion/delete_webhooks_github.py +++ b/utils/webhook_deletion/delete_webhooks_github.py @@ -1,78 +1,86 @@ import logging import pymongo import requests -from config.settings import MONGO_URI, MONGO_DB, GITHUB_TOKEN, WEBHOOK_URL_GITHUB, GITHUB_API_URL +from config.settings import ( + MONGO_URI, + MONGO_DB, + GITHUB_TOKEN, + GITHUB_API_URL, +) logger = logging.getLogger(__name__) +REQUEST_TIMEOUT = 10 - -def list_github_hooks(owner, repo)-> list: +def list_github_hooks(owner, repo) -> list: """ - The function lists all the webhooks created on a specific repository, we need the owner and the repository name. + The function lists all the webhooks created on a specific repository, we need the owner and the repository name. Also a owner or admin token is needed to authenticate the request. """ url = f"{GITHUB_API_URL}/repos/{owner}/{repo}/hooks" headers = { "Accept": "application/vnd.github.v3+json", - "Authorization": f"Bearer {GITHUB_TOKEN}" + "Authorization": f"Bearer {GITHUB_TOKEN}", } - resp = requests.get(url, headers=headers) + resp = requests.get(url, headers=headers, timeout=REQUEST_TIMEOUT) resp.raise_for_status() return resp.json() def delete_github_hook(owner, repo, hook_id): - ''' - This function deletes a specific webhook, we need the owner, the repository name and the id of the webhook to delete. + """ + This function deletes a specific webhook, we need the owner, the repository name and the id of the webhook to delete. Also a owner or admin token is needed to authenticate the request. - ''' + """ url = f"{GITHUB_API_URL}/repos/{owner}/{repo}/hooks/{hook_id}" headers = { "Accept": "application/vnd.github.v3+json", - "Authorization": f"Bearer {GITHUB_TOKEN}" + "Authorization": f"Bearer {GITHUB_TOKEN}", } - resp = requests.delete(url, headers=headers) + resp = requests.delete(url, headers=headers, timeout=REQUEST_TIMEOUT) resp.raise_for_status() return resp - def delete_all_github_webhooks(webhook_url_github): - ''' + """ This function deletes all the webhooks created on the repositories of the database. - ''' - mongo_client = pymongo.MongoClient(MONGO_URI) # URI to connect to the database - db = mongo_client[MONGO_DB] # Name of the database + """ + mongo_client = pymongo.MongoClient(MONGO_URI) # URI to connect to the database + db = mongo_client[MONGO_DB] # Name of the database # We get all collections with 'commits' in their name and store them in a list. From them we will extract the repositories names and owners. all_collections = db.list_collection_names() github_commit_collections = [c for c in all_collections if "commit" in c] - - - #We iterate over the collections and the repositories to delete the webhooks - #We first get the repositories in the collection and then we get the hooks for each repository + # We iterate over the collections and the repositories to delete the webhooks + # We first get the repositories in the collection and then we get the hooks for each repository for col_name in github_commit_collections: distinct_repos = db[col_name].distinct("repository") - + for repo_full_name in distinct_repos: - owner, repo = repo_full_name.split('/') + owner, repo = repo_full_name.split("/") try: hooks = list_github_hooks(owner, repo) logger.info("Found %d hooks for %s/%s", len(hooks), owner, repo) except requests.HTTPError as e: logger.error("Error listing hooks for %s/%s: %s", owner, repo, e) continue - + for hook in hooks: config_url = hook.get("config", {}).get("url", "") hook_id = hook.get("id") - + if config_url == webhook_url_github: # Delete it try: delete_github_hook(owner, repo, hook_id) logger.info("Deleted hook %s for %s/%s", hook_id, owner, repo) except requests.HTTPError as e: - logger.error("Error deleting hook %s for %s/%s: %s", hook_id, owner, repo, e) \ No newline at end of file + logger.error( + "Error deleting hook %s for %s/%s: %s", + hook_id, + owner, + repo, + e, + ) diff --git a/utils/webhook_deletion/delete_webhooks_taiga.py b/utils/webhook_deletion/delete_webhooks_taiga.py index d20b23d..0339a29 100644 --- a/utils/webhook_deletion/delete_webhooks_taiga.py +++ b/utils/webhook_deletion/delete_webhooks_taiga.py @@ -1,54 +1,47 @@ import logging import pymongo import requests -from config.settings import MONGO_URI, MONGO_DB, WEBHOOK_URL_TAIGA +from config.settings import MONGO_URI, MONGO_DB logger = logging.getLogger(__name__) +REQUEST_TIMEOUT = 10 - -#We define two functions to list and delete the webhooks using the API of github. -#The first function lists all the webhooks created on a repository, we need the owner and the repository name. Also a owner or admin token is needed to authenticate the request. +# We define two functions to list and delete the webhooks using the API of github. +# The first function lists all the webhooks created on a repository, we need the owner and the repository name. Also a owner or admin token is needed to authenticate the request. def list_taiga_hooks(project_id, token): url = f"https://api.taiga.io/api/v1/webhooks?project={project_id}" - headers = { - "Authorization": f"Bearer {token}" - } - resp = requests.get(url, headers=headers) + headers = {"Authorization": f"Bearer {token}"} + resp = requests.get(url, headers=headers, timeout=REQUEST_TIMEOUT) resp.raise_for_status() return resp.json() -#The second function deletes a webhook, we need the owner, the repository name and the id of the webhook to delete. + +# The second function deletes a webhook, we need the owner, the repository name and the id of the webhook to delete. # Also a owner or admin token is needed to authenticate the request. def delete_taiga_hook(hook_id, token): TAIGA_API_URL = "https://api.taiga.io" url = f"{TAIGA_API_URL}/api/v1/webhooks/{hook_id}" - headers = { - "Authorization": f"Bearer {token}" - } - resp = requests.delete(url, headers=headers) + headers = {"Authorization": f"Bearer {token}"} + resp = requests.delete(url, headers=headers, timeout=REQUEST_TIMEOUT) resp.raise_for_status() return resp - - - def delete_all_taiga_webhooks(token, webhook_url_taiga): - - mongo_client = pymongo.MongoClient(MONGO_URI) # URI to connect to the database - db = mongo_client[MONGO_DB] # Name of the database + mongo_client = pymongo.MongoClient(MONGO_URI) # URI to connect to the database + db = mongo_client[MONGO_DB] # Name of the database # We get all collections with 'epic' in their name and store them in a list. From them we will extract the projectsid of all the taiga projects active. all_collections = db.list_collection_names() taiga_collections = [c for c in all_collections if "epic" in c] - - #We iterate over the collections and the repositories to delete the webhooks - #We first get the repositories in the collection and then we get the hooks for each repository + + # We iterate over the collections and the repositories to delete the webhooks + # We first get the repositories in the collection and then we get the hooks for each repository for col_name in taiga_collections: distinct_project_ids = db[col_name].distinct("project_id") - + for project_id in distinct_project_ids: hooks = list_taiga_hooks(project_id, token) @@ -58,6 +51,10 @@ def delete_all_taiga_webhooks(token, webhook_url_taiga): if hook["url"] == webhook_url_taiga: try: delete_taiga_hook(hook["id"], token) - logger.info("Deleted taiga hook %s from project %s", hook['id'], project_id) + logger.info( + "Deleted taiga hook %s from project %s", + hook["id"], + project_id, + ) except requests.HTTPError as e: - logger.error("Error deleting taiga hook %s: %s", hook['id'], e) \ No newline at end of file + logger.error("Error deleting taiga hook %s: %s", hook["id"], e)