diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 403bd7e..26f3359 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,7 +108,7 @@ jobs: sudo chmod +x /usr/local/bin/gitleaks - name: Run Gitleaks - run: gitleaks detect --source . --redact --verbose --baseline-path .gitleaks-baseline.json --exit-code 1 + run: gitleaks detect --source . --config .gitleaks.toml --baseline-path .gitleaks-baseline.json --redact --verbose --exit-code 1 semgrep: runs-on: ubuntu-latest diff --git a/.gitleaks-baseline.json b/.gitleaks-baseline.json index afbd011..9d6c2a4 100644 --- a/.gitleaks-baseline.json +++ b/.gitleaks-baseline.json @@ -1,4 +1,109 @@ [ + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 56, + "EndLine": 56, + "StartColumn": 6, + "EndColumn": 36, + "Match": "key: ed25519.Ed25519PrivateKey\r", + "Secret": "ed25519.Ed25519PrivateKey", + "File": "myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py", + "SymlinkFile": "", + "Commit": "134dbe4b3dc21ebbb10e467674a4f07ffb296698", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/134dbe4b3dc21ebbb10e467674a4f07ffb296698/myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py#L56", + "Entropy": 3.8136606, + "Author": "sergio-utrillaa", + "Email": "sergio.utrilla@estudiantat.upc.edu", + "Date": "2026-03-26T18:35:14Z", + "Message": "Add User Story Patterns detection", + "Tags": [], + "Fingerprint": "134dbe4b3dc21ebbb10e467674a4f07ffb296698:myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py:generic-api-key:56" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 57, + "EndLine": 57, + "StartColumn": 6, + "EndColumn": 41, + "Match": "key_cls = ed25519.Ed25519PrivateKey\r", + "Secret": "ed25519.Ed25519PrivateKey", + "File": "myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py", + "SymlinkFile": "", + "Commit": "134dbe4b3dc21ebbb10e467674a4f07ffb296698", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/134dbe4b3dc21ebbb10e467674a4f07ffb296698/myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py#L57", + "Entropy": 3.8136606, + "Author": "sergio-utrillaa", + "Email": "sergio.utrilla@estudiantat.upc.edu", + "Date": "2026-03-26T18:35:14Z", + "Message": "Add User Story Patterns detection", + "Tags": [], + "Fingerprint": "134dbe4b3dc21ebbb10e467674a4f07ffb296698:myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py:generic-api-key:57" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 68, + "EndLine": 68, + "StartColumn": 6, + "EndColumn": 32, + "Match": "key: ed448.Ed448PrivateKey\r", + "Secret": "ed448.Ed448PrivateKey", + "File": "myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py", + "SymlinkFile": "", + "Commit": "134dbe4b3dc21ebbb10e467674a4f07ffb296698", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/134dbe4b3dc21ebbb10e467674a4f07ffb296698/myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py#L68", + "Entropy": 3.5944657, + "Author": "sergio-utrillaa", + "Email": "sergio.utrilla@estudiantat.upc.edu", + "Date": "2026-03-26T18:35:14Z", + "Message": "Add User Story Patterns detection", + "Tags": [], + "Fingerprint": "134dbe4b3dc21ebbb10e467674a4f07ffb296698:myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py:generic-api-key:68" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 69, + "EndLine": 69, + "StartColumn": 6, + "EndColumn": 37, + "Match": "key_cls = ed448.Ed448PrivateKey\r", + "Secret": "ed448.Ed448PrivateKey", + "File": "myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py", + "SymlinkFile": "", + "Commit": "134dbe4b3dc21ebbb10e467674a4f07ffb296698", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/134dbe4b3dc21ebbb10e467674a4f07ffb296698/myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py#L69", + "Entropy": 3.5944657, + "Author": "sergio-utrillaa", + "Email": "sergio.utrilla@estudiantat.upc.edu", + "Date": "2026-03-26T18:35:14Z", + "Message": "Add User Story Patterns detection", + "Tags": [], + "Fingerprint": "134dbe4b3dc21ebbb10e467674a4f07ffb296698:myenv/Lib/site-packages/dns/dnssecalgs/eddsa.py:generic-api-key:69" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 43, + "EndLine": 43, + "StartColumn": 10, + "EndColumn": 42, + "Match": "RegEnumKey = win32api.RegEnumKey\r", + "Secret": "win32api.RegEnumKey", + "File": "myenv/Lib/site-packages/setuptools/_distutils/msvccompiler.py", + "SymlinkFile": "", + "Commit": "134dbe4b3dc21ebbb10e467674a4f07ffb296698", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/134dbe4b3dc21ebbb10e467674a4f07ffb296698/myenv/Lib/site-packages/setuptools/_distutils/msvccompiler.py#L43", + "Entropy": 3.932138, + "Author": "sergio-utrillaa", + "Email": "sergio.utrilla@estudiantat.upc.edu", + "Date": "2026-03-26T18:35:14Z", + "Message": "Add User Story Patterns detection", + "Tags": [], + "Fingerprint": "134dbe4b3dc21ebbb10e467674a4f07ffb296698:myenv/Lib/site-packages/setuptools/_distutils/msvccompiler.py:generic-api-key:43" + }, { "RuleID": "github-pat", "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", @@ -20,48 +125,6 @@ "Tags": [], "Fingerprint": "51732615a354fe996d6892942eb0fd631c7570b3:config_files/credentials_config.json:github-pat:10" }, - { - "RuleID": "jwt", - "Description": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", - "StartLine": 36, - "EndLine": 36, - "StartColumn": 11, - "EndColumn": 522, - "Match": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQ3MzMxMDk0LCJqdGkiOiI4N2U1NWM5MTNmMzU0NjQwODU1MTFiMWMwY2I0YzM3NSIsInVzZXJfaWQiOjc1OTg5Nn0.Rh57RIdpOZLQ_xg_5c22dLIp3yXvvWB-aC2RVwgLdvSOirkhIBgqDKBXQ3j3OlbEi4BIgD-WfZZR6CXtyewOnX8ov4RCtTtdpxpuo8lch4ZhqPuvZ-UT-w8ytenrcxZoH3vz3ikUaevYDbjuCV3FSoiWn1Xxcg8jdiu-bsx-nenZ7GhydvE6VCKogF29bPjLUuZbkk-BtxVHiTPDEb6qOWx7wo83b4Io8D0zaKxgVQRzliUUy-my8HdWTex-ELyaIwzWVAkzbYGh7DjmRY4opGqFovmDkOCCOmv8Ycm3VU2RqFd7nfJAEZayMkwe1l481dvmPKMfcJ0llORocbkC0A\"", - "Secret": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQ3MzMxMDk0LCJqdGkiOiI4N2U1NWM5MTNmMzU0NjQwODU1MTFiMWMwY2I0YzM3NSIsInVzZXJfaWQiOjc1OTg5Nn0.Rh57RIdpOZLQ_xg_5c22dLIp3yXvvWB-aC2RVwgLdvSOirkhIBgqDKBXQ3j3OlbEi4BIgD-WfZZR6CXtyewOnX8ov4RCtTtdpxpuo8lch4ZhqPuvZ-UT-w8ytenrcxZoH3vz3ikUaevYDbjuCV3FSoiWn1Xxcg8jdiu-bsx-nenZ7GhydvE6VCKogF29bPjLUuZbkk-BtxVHiTPDEb6qOWx7wo83b4Io8D0zaKxgVQRzliUUy-my8HdWTex-ELyaIwzWVAkzbYGh7DjmRY4opGqFovmDkOCCOmv8Ycm3VU2RqFd7nfJAEZayMkwe1l481dvmPKMfcJ0llORocbkC0A", - "File": "utils/taiga_get_milestone_points.py", - "SymlinkFile": "", - "Commit": "81c19ba5b4da57e7b234a97a75369c537a122880", - "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/81c19ba5b4da57e7b234a97a75369c537a122880/utils/taiga_get_milestone_points.py#L36", - "Entropy": 5.883454, - "Author": "Pablo Gomez", - "Email": "pgomezna@gmail.com", - "Date": "2025-05-19T23:42:51Z", - "Message": "milestone_points", - "Tags": [], - "Fingerprint": "81c19ba5b4da57e7b234a97a75369c537a122880:utils/taiga_get_milestone_points.py:jwt:36" - }, - { - "RuleID": "jwt", - "Description": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", - "StartLine": 37, - "EndLine": 37, - "StartColumn": 11, - "EndColumn": 522, - "Match": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQ3MzA3MDI1LCJqdGkiOiIwMzA5NDcyNzZlMjc0YzQxYWM0N2NiYTllNzE2NGYyMyIsInVzZXJfaWQiOjc1OTg5Nn0.bYh7cfGv6WzhpdQB1nXu75oEI3uklWvEQAkoItY5R9j0WPutzVcXbAbeXkBH-sfJa6k-QjVCKfFYruqGSH4819q7tCYzc67sqgiA0MTsJkKuzUa_2aa1owyHMtiDYGMK3ZhN8W7KQxarMEvHXEtyrUKBI5gU_ewUoqtLBcplCBSFfrIvC9UqjA7OzS3YlBaS9YCYP3vt0ndg_qGRq1hqb64sByx7eld_6z-1Rm2KmfqHztqj6miR4K3KhNlO45lyNti_WE4nZJxfku4yXo8G91MSVFQSCZgLXUvRL-DO0b28Fu6LQX5T9or3cDNvuaGuT0SP3A1Jf-KIYU21sWYnrA\"", - "Secret": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQ3MzA3MDI1LCJqdGkiOiIwMzA5NDcyNzZlMjc0YzQxYWM0N2NiYTllNzE2NGYyMyIsInVzZXJfaWQiOjc1OTg5Nn0.bYh7cfGv6WzhpdQB1nXu75oEI3uklWvEQAkoItY5R9j0WPutzVcXbAbeXkBH-sfJa6k-QjVCKfFYruqGSH4819q7tCYzc67sqgiA0MTsJkKuzUa_2aa1owyHMtiDYGMK3ZhN8W7KQxarMEvHXEtyrUKBI5gU_ewUoqtLBcplCBSFfrIvC9UqjA7OzS3YlBaS9YCYP3vt0ndg_qGRq1hqb64sByx7eld_6z-1Rm2KmfqHztqj6miR4K3KhNlO45lyNti_WE4nZJxfku4yXo8G91MSVFQSCZgLXUvRL-DO0b28Fu6LQX5T9or3cDNvuaGuT0SP3A1Jf-KIYU21sWYnrA", - "File": "test_points.py", - "SymlinkFile": "", - "Commit": "81c19ba5b4da57e7b234a97a75369c537a122880", - "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/81c19ba5b4da57e7b234a97a75369c537a122880/test_points.py#L37", - "Entropy": 5.9098024, - "Author": "Pablo Gomez", - "Email": "pgomezna@gmail.com", - "Date": "2025-05-19T23:42:51Z", - "Message": "milestone_points", - "Tags": [], - "Fingerprint": "81c19ba5b4da57e7b234a97a75369c537a122880:test_points.py:jwt:37" - }, { "RuleID": "github-pat", "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", @@ -104,6 +167,69 @@ "Tags": [], "Fingerprint": "4c29834f8242fe60c28182f982df7c9616923cf4:test.py:github-pat:40" }, + { + "RuleID": "github-pat", + "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "StartLine": 24, + "EndLine": 24, + "StartColumn": 20, + "EndColumn": 59, + "Match": "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", + "Secret": "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", + "File": "recovery/github_recovery.py", + "SymlinkFile": "", + "Commit": "4c29834f8242fe60c28182f982df7c9616923cf4", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/4c29834f8242fe60c28182f982df7c9616923cf4/recovery/github_recovery.py#L24", + "Entropy": 4.803056, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-05-27T00:26:14Z", + "Message": "recovry", + "Tags": [], + "Fingerprint": "4c29834f8242fe60c28182f982df7c9616923cf4:recovery/github_recovery.py:github-pat:24" + }, + { + "RuleID": "jwt", + "Description": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", + "StartLine": 36, + "EndLine": 36, + "StartColumn": 11, + "EndColumn": 522, + "Match": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQ3MzMxMDk0LCJqdGkiOiI4N2U1NWM5MTNmMzU0NjQwODU1MTFiMWMwY2I0YzM3NSIsInVzZXJfaWQiOjc1OTg5Nn0.Rh57RIdpOZLQ_xg_5c22dLIp3yXvvWB-aC2RVwgLdvSOirkhIBgqDKBXQ3j3OlbEi4BIgD-WfZZR6CXtyewOnX8ov4RCtTtdpxpuo8lch4ZhqPuvZ-UT-w8ytenrcxZoH3vz3ikUaevYDbjuCV3FSoiWn1Xxcg8jdiu-bsx-nenZ7GhydvE6VCKogF29bPjLUuZbkk-BtxVHiTPDEb6qOWx7wo83b4Io8D0zaKxgVQRzliUUy-my8HdWTex-ELyaIwzWVAkzbYGh7DjmRY4opGqFovmDkOCCOmv8Ycm3VU2RqFd7nfJAEZayMkwe1l481dvmPKMfcJ0llORocbkC0A\"", + "Secret": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQ3MzMxMDk0LCJqdGkiOiI4N2U1NWM5MTNmMzU0NjQwODU1MTFiMWMwY2I0YzM3NSIsInVzZXJfaWQiOjc1OTg5Nn0.Rh57RIdpOZLQ_xg_5c22dLIp3yXvvWB-aC2RVwgLdvSOirkhIBgqDKBXQ3j3OlbEi4BIgD-WfZZR6CXtyewOnX8ov4RCtTtdpxpuo8lch4ZhqPuvZ-UT-w8ytenrcxZoH3vz3ikUaevYDbjuCV3FSoiWn1Xxcg8jdiu-bsx-nenZ7GhydvE6VCKogF29bPjLUuZbkk-BtxVHiTPDEb6qOWx7wo83b4Io8D0zaKxgVQRzliUUy-my8HdWTex-ELyaIwzWVAkzbYGh7DjmRY4opGqFovmDkOCCOmv8Ycm3VU2RqFd7nfJAEZayMkwe1l481dvmPKMfcJ0llORocbkC0A", + "File": "utils/taiga_get_milestone_points.py", + "SymlinkFile": "", + "Commit": "81c19ba5b4da57e7b234a97a75369c537a122880", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/81c19ba5b4da57e7b234a97a75369c537a122880/utils/taiga_get_milestone_points.py#L36", + "Entropy": 5.883454, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-05-19T23:42:51Z", + "Message": "milestone_points", + "Tags": [], + "Fingerprint": "81c19ba5b4da57e7b234a97a75369c537a122880:utils/taiga_get_milestone_points.py:jwt:36" + }, + { + "RuleID": "jwt", + "Description": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", + "StartLine": 37, + "EndLine": 37, + "StartColumn": 11, + "EndColumn": 522, + "Match": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQ3MzA3MDI1LCJqdGkiOiIwMzA5NDcyNzZlMjc0YzQxYWM0N2NiYTllNzE2NGYyMyIsInVzZXJfaWQiOjc1OTg5Nn0.bYh7cfGv6WzhpdQB1nXu75oEI3uklWvEQAkoItY5R9j0WPutzVcXbAbeXkBH-sfJa6k-QjVCKfFYruqGSH4819q7tCYzc67sqgiA0MTsJkKuzUa_2aa1owyHMtiDYGMK3ZhN8W7KQxarMEvHXEtyrUKBI5gU_ewUoqtLBcplCBSFfrIvC9UqjA7OzS3YlBaS9YCYP3vt0ndg_qGRq1hqb64sByx7eld_6z-1Rm2KmfqHztqj6miR4K3KhNlO45lyNti_WE4nZJxfku4yXo8G91MSVFQSCZgLXUvRL-DO0b28Fu6LQX5T9or3cDNvuaGuT0SP3A1Jf-KIYU21sWYnrA\"", + "Secret": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzQ3MzA3MDI1LCJqdGkiOiIwMzA5NDcyNzZlMjc0YzQxYWM0N2NiYTllNzE2NGYyMyIsInVzZXJfaWQiOjc1OTg5Nn0.bYh7cfGv6WzhpdQB1nXu75oEI3uklWvEQAkoItY5R9j0WPutzVcXbAbeXkBH-sfJa6k-QjVCKfFYruqGSH4819q7tCYzc67sqgiA0MTsJkKuzUa_2aa1owyHMtiDYGMK3ZhN8W7KQxarMEvHXEtyrUKBI5gU_ewUoqtLBcplCBSFfrIvC9UqjA7OzS3YlBaS9YCYP3vt0ndg_qGRq1hqb64sByx7eld_6z-1Rm2KmfqHztqj6miR4K3KhNlO45lyNti_WE4nZJxfku4yXo8G91MSVFQSCZgLXUvRL-DO0b28Fu6LQX5T9or3cDNvuaGuT0SP3A1Jf-KIYU21sWYnrA", + "File": "test_points.py", + "SymlinkFile": "", + "Commit": "81c19ba5b4da57e7b234a97a75369c537a122880", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/81c19ba5b4da57e7b234a97a75369c537a122880/test_points.py#L37", + "Entropy": 5.9098024, + "Author": "Pablo Gomez", + "Email": "pgomezna@gmail.com", + "Date": "2025-05-19T23:42:51Z", + "Message": "milestone_points", + "Tags": [], + "Fingerprint": "81c19ba5b4da57e7b234a97a75369c537a122880:test_points.py:jwt:37" + }, { "RuleID": "github-pat", "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", @@ -170,52 +296,31 @@ { "RuleID": "github-pat", "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", - "StartLine": 24, - "EndLine": 24, - "StartColumn": 20, - "EndColumn": 59, - "Match": "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", - "Secret": "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", - "File": "recovery/github_recovery.py", - "SymlinkFile": "", - "Commit": "4c29834f8242fe60c28182f982df7c9616923cf4", - "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/4c29834f8242fe60c28182f982df7c9616923cf4/recovery/github_recovery.py#L24", - "Entropy": 4.803056, - "Author": "Pablo Gomez", - "Email": "pgomezna@gmail.com", - "Date": "2025-05-27T00:26:14Z", - "Message": "recovry", - "Tags": [], - "Fingerprint": "4c29834f8242fe60c28182f982df7c9616923cf4:recovery/github_recovery.py:github-pat:24" - }, - { - "RuleID": "github-pat", - "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", - "StartLine": 10, - "EndLine": 10, - "StartColumn": 44, - "EndColumn": 83, + "StartLine": 7, + "EndLine": 7, + "StartColumn": 28, + "EndColumn": 67, "Match": "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", "Secret": "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", "File": "datasources/github_handler.py", "SymlinkFile": "", - "Commit": "fb5af68da7c610cb306db0db52fdae2473fa7a28", - "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/fb5af68da7c610cb306db0db52fdae2473fa7a28/datasources/github_handler.py#L10", + "Commit": "429d462f6209cc32022bb0d8ecab3052c16ed1d7", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/429d462f6209cc32022bb0d8ecab3052c16ed1d7/datasources/github_handler.py#L7", "Entropy": 4.8341837, "Author": "Pablo Gomez", "Email": "pgomezna@gmail.com", - "Date": "2025-03-26T15:54:51Z", - "Message": "Solving problems with API call to commit stats", + "Date": "2025-03-26T13:02:39Z", + "Message": "Commit Stats First implementation", "Tags": [], - "Fingerprint": "fb5af68da7c610cb306db0db52fdae2473fa7a28:datasources/github_handler.py:github-pat:10" + "Fingerprint": "429d462f6209cc32022bb0d8ecab3052c16ed1d7:datasources/github_handler.py:github-pat:7" }, { "RuleID": "github-pat", "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", "StartLine": 7, "EndLine": 7, - "StartColumn": 28, - "EndColumn": 67, + "StartColumn": 72, + "EndColumn": 111, "Match": "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", "Secret": "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", "File": "datasources/github_handler.py", @@ -233,23 +338,23 @@ { "RuleID": "github-pat", "Description": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", - "StartLine": 7, - "EndLine": 7, - "StartColumn": 72, - "EndColumn": 111, + "StartLine": 10, + "EndLine": 10, + "StartColumn": 44, + "EndColumn": 83, "Match": "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", "Secret": "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", "File": "datasources/github_handler.py", "SymlinkFile": "", - "Commit": "429d462f6209cc32022bb0d8ecab3052c16ed1d7", - "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/429d462f6209cc32022bb0d8ecab3052c16ed1d7/datasources/github_handler.py#L7", + "Commit": "fb5af68da7c610cb306db0db52fdae2473fa7a28", + "Link": "https://github.com/Learning-Dashboard/LD_Connect_Event/blob/fb5af68da7c610cb306db0db52fdae2473fa7a28/datasources/github_handler.py#L10", "Entropy": 4.8341837, "Author": "Pablo Gomez", "Email": "pgomezna@gmail.com", - "Date": "2025-03-26T13:02:39Z", - "Message": "Commit Stats First implementation", + "Date": "2025-03-26T15:54:51Z", + "Message": "Solving problems with API call to commit stats", "Tags": [], - "Fingerprint": "429d462f6209cc32022bb0d8ecab3052c16ed1d7:datasources/github_handler.py:github-pat:7" + "Fingerprint": "fb5af68da7c610cb306db0db52fdae2473fa7a28:datasources/github_handler.py:github-pat:10" }, { "RuleID": "github-pat", diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000..891401b --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,10 @@ +title = "Gitleaks config" + +[extend] +useDefault = true + +[[allowlists]] +description = "Ignore generated Gitleaks baseline report" +paths = [ + '''^\.gitleaks-baseline\.json$''' +] diff --git a/app.py b/app.py index a1de4fc..4a53153 100644 --- a/app.py +++ b/app.py @@ -63,6 +63,10 @@ def health(): app.register_blueprint(github_bp) app.register_blueprint(taiga_bp) app.register_blueprint(excel_bp) + + app.register_blueprint(github_bp, url_prefix="/webhooks", name="github_bp_prefixed") + app.register_blueprint(taiga_bp, url_prefix="/webhooks", name="taiga_bp_prefixed") + app.register_blueprint(excel_bp, url_prefix="/webhooks", name="excel_bp_prefixed") _register_error_handlers(app) _log_credentials_config_status() logger.info("Flask created and Blueprints registered successfully.") diff --git a/datasources/github_handler.py b/datasources/github_handler.py index 72b35f6..31482c5 100644 --- a/datasources/github_handler.py +++ b/datasources/github_handler.py @@ -35,9 +35,10 @@ def parse_github_push_event(raw_payload: Dict, prj: str) -> Dict: # The 'sender' object is at the top level sender = raw_payload.get("sender", {}) + sender_login = sender.get("login") or "anonymous" sender_info = { "id": sender.get("id", ""), - "login": sender.get("login", ""), + "login": sender_login, "url": sender.get("url", ""), "type": sender.get("type", ""), "site_admin": sender.get("site_admin", False), @@ -54,8 +55,8 @@ def parse_github_push_event(raw_payload: Dict, prj: str) -> Dict: date = to_madrid_local(c.get("timestamp")) # Built author information - author_login = c.get("author", {}).get("username", "") - author_name = c.get("author", {}).get("name", "") + author_login = c.get("author", {}).get("username") or sender_login + author_name = c.get("author", {}).get("name", "") author_email = c.get("author", {}).get("email", "") # Compute message stats diff --git a/datasources/requests/taiga_api_call.py b/datasources/requests/taiga_api_call.py index 54bbda4..e5a433d 100644 --- a/datasources/requests/taiga_api_call.py +++ b/datasources/requests/taiga_api_call.py @@ -2,17 +2,27 @@ import requests from datetime import datetime, timedelta, timezone from utils.taiga_token.taiga_auth import get_taiga_token -from config.credentials_loader import resolve +from config.credentials_loader import ( + CredentialsConfigError, + ProjectCredentialsNotFoundError, + resolve, +) from config.settings import TAIGA_API_URL +_CACHE = {} # key = (project_id, milestone_id) -> (timestamp, stats) +_DETAILS_CACHE = {} # key = (project_id, milestone_id) -> (timestamp, details) +_USERSTORY_CACHE = {} # key = (project_id, userstory_id) -> (timestamp, details) +TTL = timedelta(minutes=1) # Cache time-to-live, set to 5 minutes. Means that if the same request is made within 5 minutes, it will return the cached result instead of making a new API call. logger = logging.getLogger(__name__) _CACHE = {} # key = (project_id, milestone_id) -> (timestamp, stats) -TTL = timedelta( - minutes=1 -) # Cache time-to-live, set to 5 minutes. Means that if the same request is made within 5 minutes, it will return the cached result instead of making a new API call. MILESTONE_TIMEOUT = (3, 8) +TAIGA_LOOKUP_ERRORS = ( + requests.exceptions.RequestException, + CredentialsConfigError, + ProjectCredentialsNotFoundError, +) def _empty_stats(): @@ -26,6 +36,96 @@ def _empty_stats(): } + +def _build_taiga_headers(prj: str): + """Return the Taiga headers needed for public and private deployments.""" + if "api.taiga.io" in TAIGA_API_URL: + user = resolve(prj, "taiga_user") + psw = resolve(prj, "taiga_password") + if user and psw: + token = get_taiga_token(user, psw) + return {"Authorization": f"Bearer {token}"} + return {} + + +def milestone_details(project_id: str, milestone_id: str, prj: str): + """ + Fetches the milestone metadata from Taiga. + Returns the raw milestone fields needed to enrich recovery documents. + """ + if not project_id or not milestone_id: + return {} + + key = (project_id, milestone_id) + now = datetime.now(timezone.utc) + if key in _DETAILS_CACHE and now - _DETAILS_CACHE[key][0] < TTL: + return _DETAILS_CACHE[key][1] + + try: + headers = _build_taiga_headers(prj) + url = f"{TAIGA_API_URL}/milestones/{milestone_id}" + r = requests.get( + url, params={"project": project_id}, headers=headers, timeout=(1, 5) + ) + r.raise_for_status() + except TAIGA_LOOKUP_ERRORS as exc: + logger.warning( + "Failed to fetch milestone details for project %s milestone %s: %s", + project_id, + milestone_id, + exc, + ) + return {} + + js = r.json() + details = { + "milestone_created_date": js.get("created_date"), + "milestone_modified_date": js.get("modified_date"), + "milestone_name": js.get("name"), + "estimated_start": js.get("estimated_start"), + "estimated_finish": js.get("estimated_finish"), + "milestone_closed": bool(js.get("closed", False)), + } + _DETAILS_CACHE[key] = (now, details) + return details + + +def userstory_details(project_id: str, userstory_id: str, prj: str): + """ + Fetches the userstory metadata from Taiga. + Used as a fallback when task payloads do not include the nested userstory state. + """ + if not project_id or not userstory_id: + return {} + + key = (project_id, userstory_id) + now = datetime.now(timezone.utc) + if key in _USERSTORY_CACHE and now - _USERSTORY_CACHE[key][0] < TTL: + return _USERSTORY_CACHE[key][1] + + try: + headers = _build_taiga_headers(prj) + url = f"{TAIGA_API_URL}/userstories/{userstory_id}" + r = requests.get( + url, params={"project": project_id}, headers=headers, timeout=(1, 5) + ) + r.raise_for_status() + except TAIGA_LOOKUP_ERRORS as exc: + logger.warning( + "Failed to fetch user story details for project %s user story %s: %s", + project_id, + userstory_id, + exc, + ) + return {} + + js = r.json() + details = { + "userstory_is_closed": (js.get("status_extra_info") or {}).get("is_closed"), + } + _USERSTORY_CACHE[key] = (now, details) + return details + def milestone_stats(project_id: str, milestone_id: str, prj: str): """ Fetches the statistics of a milestone in a Taiga project. @@ -40,56 +140,26 @@ def milestone_stats(project_id: str, milestone_id: str, prj: str): if key in _CACHE and now - _CACHE[key][0] < TTL: return _CACHE[key][1] - user = resolve(prj, "taiga_user") - psw = resolve(prj, "taiga_password") - logger.debug( - "Resolving Taiga credentials for project %s: user=%s, password=%s", - prj, - "****" if user else None, - "****" if psw else None, - ) - if user and psw: - try: - token = get_taiga_token(user, psw) - except requests.exceptions.RequestException as exc: - logger.warning( - "Failed to get Taiga token for project %s. Returning empty milestone stats: %s", - prj, - exc, - ) - stats = _empty_stats() - _CACHE[key] = (now, stats) - return stats - - headers = {"Authorization": f"Bearer {token}"} - logger.debug("Using Taiga credentials for project %s", prj) - else: - headers = {} - logger.info("Using Taiga tunnel without authentication for project: %s", prj) - - url = f"{TAIGA_API_URL}/milestones/{milestone_id}/stats" - logger.debug("Fetching Taiga milestone stats from URL: %s", url) try: + headers = _build_taiga_headers(prj) + url = f"{TAIGA_API_URL}/milestones/{milestone_id}/stats" r = requests.get( - url, params={"project": project_id}, headers=headers, timeout=MILESTONE_TIMEOUT + url, params={"project": project_id}, headers=headers, timeout=(1, 5) ) r.raise_for_status() - except requests.exceptions.RequestException as exc: + except TAIGA_LOOKUP_ERRORS as exc: logger.warning( - "Failed to fetch milestone stats for project %s milestone %s. " - "Returning empty milestone stats: %s", - prj, + "Failed to fetch milestone stats for project %s milestone %s: %s", + project_id, milestone_id, exc, ) - stats = _empty_stats() - _CACHE[key] = (now, stats) - return stats + return _empty_stats() js = r.json() stats = { - "milestone_total_points": sum(js.get("total_points", {}).values()), - "milestone_closed_points": sum(js.get("completed_points", 0)), + "milestone_total_points": sum((js.get("total_points") or {}).values()), + "milestone_closed_points": sum(js.get("completed_points") or []), "milestone_total_userstories": js.get("total_userstories", 0), "milestone_completed_userstories": js.get("completed_userstories", 0), "milestone_total_tasks": js.get("total_tasks", 0), diff --git a/datasources/taiga_handler.py b/datasources/taiga_handler.py index 486cd16..f411c73 100644 --- a/datasources/taiga_handler.py +++ b/datasources/taiga_handler.py @@ -1,5 +1,5 @@ from typing import Dict -import re +from utils.pattern_detector import PatternDetector from datasources.requests.taiga_api_call import milestone_stats from utils.datetime_utils import to_madrid_local @@ -141,36 +141,18 @@ def parse_taiga_task_event(raw_payload: Dict, prj: str) -> Dict: is_closed = raw_payload.get("data", {}).get("status", {}).get("is_closed", "") status = raw_payload.get("data", {}).get("status", {}).get("name", "") created_date = to_madrid_local(raw_payload.get("data", {}).get("created_date", "")) - modified_date = to_madrid_local( - raw_payload.get("data", {}).get("modified_date", "") - ) - finished_date = to_madrid_local( - raw_payload.get("data", {}).get("finished_date", "") - ) - reference = raw_payload.get("data", {}).get("ref", "") - milestone_id = raw_payload.get("data", {}).get("milestone", {}).get("id", "") - milestone_name = raw_payload.get("data", {}).get("milestone", {}).get("name", "") - milestone_closed = ( - raw_payload.get("data", {}).get("milestone", {}).get("closed", "") - ) - milestone_created_date = ( - raw_payload.get("data", {}).get("milestone", {}).get("created_date", "") - ) - milestone_created_date = ( - to_madrid_local(milestone_created_date) if milestone_created_date else "" - ) - milestone_modified_date = ( - raw_payload.get("data", {}).get("milestone", {}).get("modified_date", "") - ) - milestone_modified_date = ( - to_madrid_local(milestone_modified_date) if milestone_modified_date else "" - ) - estimated_start = to_madrid_local( - raw_payload.get("data", {}).get("milestone", {}).get("estimated_start", "") - ) - estimated_finish = to_madrid_local( - raw_payload.get("data", {}).get("milestone", {}).get("estimated_finish", "") - ) + modified_date = to_madrid_local(raw_payload.get("data", {}).get("modified_date", "")) + finished_date = to_madrid_local(raw_payload.get("data", {}).get("finished_date", "")) + reference=raw_payload.get("data",{}).get("ref", "") + milestone_id=raw_payload.get("data",{}).get("milestone",{}).get("id", "") + milestone_name=raw_payload.get("data",{}).get("milestone",{}).get("name", "") + milestone_closed=bool(raw_payload.get("data",{}).get("milestone",{}).get("closed", False)) + milestone_created_date=raw_payload.get("data",{}).get("milestone",{}).get("created_date", "") + milestone_created_date = to_madrid_local(milestone_created_date) if milestone_created_date else "" + milestone_modified_date=raw_payload.get("data",{}).get("milestone",{}).get("modified_date", "") + milestone_modified_date = to_madrid_local(milestone_modified_date) if milestone_modified_date else "" + estimated_start=to_madrid_local(raw_payload.get("data",{}).get("milestone",{}).get("estimated_start", "")) + estimated_finish=to_madrid_local(raw_payload.get("data",{}).get("milestone",{}).get("estimated_finish", "")) assigned_by = raw_payload.get("by", {}).get("username", "") milestone_data = milestone_stats(project_id, milestone_id, prj) @@ -244,14 +226,36 @@ def parse_taiga_userstory_event(raw_payload: Dict, prj: str) -> Dict: if custom_attributes is None: custom_attributes = {} - description = raw_payload.get("data", {}).get("description", "") - # If the pattern "AS - A - I WANT - SO THAT" is used in the description, the vañue of pattern will be True, if not, it will be False - pattern = r"as\s+(.*?)\s+i want\s+(.*?)\s+so that\s+(.*)" - match = re.search(pattern, description, re.IGNORECASE) - if match: - pattern_in_description = True + description= raw_payload.get("data", {}).get("description", "") + # Detect BDD pattern (EN/ES/CA) + pattern_in_description = PatternDetector.detect_pattern(description) + + # If the userstory has a milestone associated while created, we will get the values, if not, we will set them to None + if raw_payload.get("data",{}).get("milestone",{}) is not None: + + milestone_id= raw_payload.get("data",{}).get("milestone",{}).get("id", "") + milestone_name= raw_payload.get("data",{}).get("milestone",{}).get("name", "") + milestone_closed= bool(raw_payload.get("data",{}).get("milestone",{}).get("closed", False)) + milestone_created_date= to_madrid_local(raw_payload.get("data",{}).get("milestone",{}).get("created_date", "")) + milestone_modified_date= to_madrid_local(raw_payload.get("data",{}).get("milestone",{}).get("modified_date", "")) + milestone_modified_date = to_madrid_local(milestone_modified_date) if milestone_modified_date else "" + estimated_start= to_madrid_local(raw_payload.get("data",{}).get("milestone",{}).get("estimated_start", "")) + estimated_finish= to_madrid_local(raw_payload.get("data",{}).get("milestone",{}).get("estimated_finish", "")) + + milestone_data= milestone_stats(project_id, milestone_id, prj) + + + else: - pattern_in_description = False + milestone_id= "" + milestone_name= "" + milestone_closed= False + milestone_created_date= "" + milestone_modified_date= "" + estimated_start= "" + estimated_finish= "" + milestone_data = {} + # If the userstory has a milestone associated while created, we will get the values, if not, we will set them to None if raw_payload.get("data", {}).get("milestone", {}) is not None: diff --git a/utils/pattern_detector.py b/utils/pattern_detector.py new file mode 100644 index 0000000..d2b180d --- /dev/null +++ b/utils/pattern_detector.py @@ -0,0 +1,47 @@ +import re + +class PatternDetector: + """ + Centralizado detector de patrones de User Stories multilenguaje. + Soporta: EN, ES, CA con sus variantes. + """ + + # Patrones regex compilados para optimización + PATTERNS = [ + # English + r"\bas\s+[\w\s]+\s+i\s+want\s+[\w\s,.:;!?-]+\s+so\s+that\s+[\w\s,.:;!?-]+", + r"\bas\s+[\w\s]+\s+i\s+want\s+[\w\s,.:;!?-]+\s+to\s+[\w\s,.:;!?-]+", + + # Spanish - COMO...QUIERO... + r"\bcomo\s+[\w\s]+\s+quiero\s+[\w\s,.:;!?-]+\s+(?:de\s+manera\s+que|de\s+forma\s+que|para|por|porqu[eé]|porque)\s+[\w\s,.:;!?-]+", + + # Catalan - COM...VULL... + r"\bcom\s+[\w\s]+\s+vull\s+[\w\s,.:;!?-]+\s+(?:de\s+manera\s+que|de\s+forma\s+que|per|perqu[eè]|perqué)\s+[\w\s,.:;!?-]+", + ] + + # Compilar patrones una sola vez + _compiled_patterns = [re.compile(p, re.IGNORECASE) for p in PATTERNS] + + @classmethod + def detect_pattern(cls, description: str) -> bool: + """ + Detecta si una descripción contiene alguno de los patrones BDD soportados. + + Args: + description: Texto de descripción de user story + + Returns: + bool: True si contiene patrón válido, False en caso contrario + """ + if not description or not isinstance(description, str): + return False + + # Normalizar: eliminar saltos de línea excesivos, mantener separadores + normalized = ' '.join(description.split()) + + # Probar cada patrón + for pattern in cls._compiled_patterns: + if pattern.search(normalized): + return True + + return False \ No newline at end of file diff --git a/utils/recovery/taiga_recovery.py b/utils/recovery/taiga_recovery.py index b63cb25..16e1244 100644 --- a/utils/recovery/taiga_recovery.py +++ b/utils/recovery/taiga_recovery.py @@ -1,5 +1,4 @@ import argparse -import re import requests from pymongo import UpdateOne from datetime import datetime, timezone @@ -13,10 +12,21 @@ from config.settings import TAIGA_API_URL +from utils.pattern_detector import PatternDetector +from datasources.requests.taiga_api_call import milestone_details, milestone_stats, userstory_details + setup_logging() logger = logging.getLogger(__name__) +def first_non_empty(*values, default=""): + for value in values: + if value not in (None, ""): + return value + return default + + + def parse_dt(s: str) -> datetime: """Accepts ‘2025-05-01’, ‘2025-05-01T14:30’, etc. And returns the datetime tz-aware (Madrid).""" d = dtparser.isoparse(s) @@ -136,33 +146,49 @@ def task_from_api(j: dict, prj: str) -> dict: Converts a task JSON object from the Taiga API to a MongoDB document schema. """ m = j.get("milestone_extra_info") or {} + project_id = (j.get("project_extra_info") or {}).get("id") + milestone_id = j.get("milestone") + milestone_info = milestone_details(project_id, milestone_id, prj) + milestone_stats_data = milestone_stats(project_id, milestone_id, prj) + userstory_id = j.get("user_story") us = j.get("user_story_extra_info") or {} - return { - "task_id": j["id"], - "action_type": "import", - "assigned_by": "backfill", - "assigned_to": (j.get("assigned_to_extra_info") or {}).get("username"), - "created_date": j["created_date"], + userstory_is_closed = us.get("is_closed") + userstory_info = ( + userstory_details(project_id, userstory_id, prj) + if userstory_id and userstory_is_closed in (None, "") + else {} + ) + doc = { + "task_id": j["id"], + "action_type": "import", + "assigned_by": "backfill", + "assigned_to": (j.get("assigned_to_extra_info") or {}).get("username"), + "created_date": j["created_date"], "custom_attributes": j.get("custom_attributes_values") or {}, - "estimated_finish": m.get("estimated_finish"), - "estimated_start": m.get("estimated_start"), - "event_type": "task", - "finished_date": j["finished_date"], - "is_closed": j["status_extra_info"]["is_closed"], - "milestone_closed": m.get("closed"), - "milestone_created_date": m.get("created_date"), - "milestone_id": j.get("milestone"), - "milestone_modified_date": m.get("modified_date"), - "milestone_name": m.get("name"), - "modified_date": j["modified_date"], - "prj": prj, - "reference": j["ref"], - "status": j["status_extra_info"]["name"], - "subject": j["subject"], - "team_name": j["project_extra_info"]["name"], - "userstory_id": j.get("user_story"), - "userstory_is_closed": us.get("is_closed"), + "estimated_finish": first_non_empty(m.get("estimated_finish"), milestone_info.get("estimated_finish")), + "estimated_start": first_non_empty(m.get("estimated_start"), milestone_info.get("estimated_start")), + "event_type": "task", + "finished_date": j["finished_date"], + "is_closed": j["status_extra_info"]["is_closed"], + "milestone_closed": bool(first_non_empty(m.get("closed"), milestone_info.get("milestone_closed"), False)), + "milestone_created_date": first_non_empty(m.get("created_date"), milestone_info.get("milestone_created_date")), + "milestone_id": milestone_id, + "milestone_modified_date": first_non_empty(m.get("modified_date"), milestone_info.get("milestone_modified_date")), + "milestone_name": first_non_empty(m.get("name"), milestone_info.get("milestone_name")), + "modified_date": j["modified_date"], + "prj": prj, + "reference": j["ref"], + "status": j["status_extra_info"]["name"], + "subject": j["subject"], + "team_name": j["project_extra_info"]["name"], + "userstory_id": j.get("user_story"), + "userstory_is_closed": first_non_empty( + userstory_is_closed, userstory_info.get("userstory_is_closed") + ), } + doc.update(milestone_stats_data) + return doc + def issue_from_api(j: dict, prj: str) -> dict: @@ -216,29 +242,33 @@ def userstory_from_api(j: dict, prj: str) -> dict: Converts an userstory JSON object from the Taiga API to a MongoDB document schema. """ m = j.get("milestone_extra_info") or {} + project_id = (j.get("project_extra_info") or {}).get("id") + milestone_id = j.get("milestone") + milestone_info = milestone_details(project_id, milestone_id, prj) + milestone_stats_data = milestone_stats(project_id, milestone_id, prj) desc = j.get("description") or "" - pattern = bool(re.search(r"as\s+.*?\s+i want\s+.*?\s+so that\s+.*", desc, re.I)) - raw_points = j.get("points") # puede ser list | "" | None + pattern = PatternDetector.detect_pattern(desc) + raw_points = j.get("points") # puede ser list | "" | None if isinstance(raw_points, list): total = sum((p.get("value") or 0) for p in raw_points) else: total = 0 - return { + doc = { "userstory_id": j["id"], "action_type": "import", "assigned_by": "backfill", "created_date": j["created_date"], "custom_attributes": j.get("custom_attributes_values") or {}, - "estimated_finish": m.get("estimated_finish"), - "estimated_start": m.get("estimated_start"), - "event_type": "userstory", - "is_closed": (j.get("status_extra_info") or {}).get("is_closed"), - "milestone_closed": m.get("closed"), - "milestone_created_date": m.get("created_date"), - "milestone_id": j.get("milestone"), - "milestone_modified_date": m.get("modified_date"), - "milestone_name": m.get("name"), + "estimated_finish": first_non_empty(m.get("estimated_finish"), milestone_info.get("estimated_finish")), + "estimated_start": first_non_empty(m.get("estimated_start"), milestone_info.get("estimated_start")), + "event_type": "userstory", + "is_closed": (j.get("status_extra_info") or {}).get("is_closed"), + "milestone_closed": bool(first_non_empty(m.get("closed"), milestone_info.get("milestone_closed"), False)), + "milestone_created_date": first_non_empty(m.get("created_date"), milestone_info.get("milestone_created_date")), + "milestone_id": milestone_id, + "milestone_modified_date": first_non_empty(m.get("modified_date"), milestone_info.get("milestone_modified_date")), + "milestone_name": first_non_empty(m.get("name"), milestone_info.get("milestone_name")), "modified_date": j["modified_date"], "pattern": pattern, "priority": (j.get("custom_attributes_values") or {}).get("Priority"), @@ -248,6 +278,8 @@ def userstory_from_api(j: dict, prj: str) -> dict: "team_name": j["project_extra_info"]["name"], "total_points": total, } + doc.update(milestone_stats_data) + return doc ENTITY_ENDPOINT = { @@ -316,16 +348,12 @@ def main(argv: list[str] | None = None): total = 0 for event in events: # Iterate over the events to backfill endpoint, converter, key = ENTITY_ENDPOINT[event] - raw = fetch_entities( - event, pid, start, end - ) # Get the raw data from the Taiga API for the event - docs = [ - converter(r, ns.prj) for r in raw - ] # Convert the raw data to the MongoDB schema using the converter function - coll = get_collection( - f"taiga_{ns.prj}.{event}" - ) # Get the MongoDB collection for the event - n = upsert(coll, docs, key) # Upsert the documents + raw = fetch_entities(event, pid, start, end) # Get the raw data from the Taiga API for the event + docs = [converter(r, ns.prj) for r in raw] # Convert the raw data to the MongoDB schema using the converter function + # Usar el nombre plural correcto para la colección de userstories + collection_name = f"taiga_{ns.prj}.userstories" if event == "userstory" else f"taiga_{ns.prj}.tasks" if event == "task" else f"taiga_{ns.prj}.epics" if event == "epic" else f"taiga_{ns.prj}.{event}" + coll = get_collection(collection_name) # Get the MongoDB collection for the event + n = upsert(coll, docs, key) # Upsert the documents total += n logger.info(" • %s → %d documents", event, n)