From f1ca8e99f48084dbeb90d7dfb14c25c095094c13 Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Thu, 25 Jun 2026 12:24:01 +0100 Subject: [PATCH 1/2] fix: exclude archived repos from vulnerability data in Tinybird pipes (IN-1186) Signed-off-by: Joana Maia --- .../libs/tinybird/pipes/agentic_ai_projects_list_copy.pipe | 2 +- .../libs/tinybird/pipes/insights_projects_populated_copy.pipe | 4 ++++ .../libs/tinybird/pipes/vulnerabilities_by_ecosystem.pipe | 4 +++- services/libs/tinybird/pipes/vulnerabilities_by_severity.pipe | 4 +++- services/libs/tinybird/pipes/vulnerabilities_list.pipe | 2 ++ services/libs/tinybird/pipes/vulnerabilities_summary.pipe | 1 + 6 files changed, 14 insertions(+), 3 deletions(-) diff --git a/services/libs/tinybird/pipes/agentic_ai_projects_list_copy.pipe b/services/libs/tinybird/pipes/agentic_ai_projects_list_copy.pipe index 524e525b4d..7309e052f7 100644 --- a/services/libs/tinybird/pipes/agentic_ai_projects_list_copy.pipe +++ b/services/libs/tinybird/pipes/agentic_ai_projects_list_copy.pipe @@ -204,7 +204,7 @@ SQL > ) AS vulnerabilities30d FROM vulnerabilities v FINAL JOIN repositories r FINAL ON v.repoUrl = r.url - WHERE v.repoUrl IN (SELECT url FROM ai_repos) + WHERE v.repoUrl IN (SELECT url FROM ai_repos) AND r.archived = false GROUP BY r.insightsProjectId NODE final_join diff --git a/services/libs/tinybird/pipes/insights_projects_populated_copy.pipe b/services/libs/tinybird/pipes/insights_projects_populated_copy.pipe index 2660d6a450..d117b6fccd 100644 --- a/services/libs/tinybird/pipes/insights_projects_populated_copy.pipe +++ b/services/libs/tinybird/pipes/insights_projects_populated_copy.pipe @@ -188,6 +188,10 @@ SQL > LEFT JOIN vulnerability_scans vs final ON vs.repoUrl = insights_projects_populated_copy_flatten_projects.repository + WHERE + insights_projects_populated_copy_flatten_projects.repository NOT IN ( + SELECT url FROM repositories FINAL WHERE archived = true + ) group by insightsProjectId NODE insights_projects_populated_copy_results diff --git a/services/libs/tinybird/pipes/vulnerabilities_by_ecosystem.pipe b/services/libs/tinybird/pipes/vulnerabilities_by_ecosystem.pipe index f57030f733..34bd5da583 100644 --- a/services/libs/tinybird/pipes/vulnerabilities_by_ecosystem.pipe +++ b/services/libs/tinybird/pipes/vulnerabilities_by_ecosystem.pipe @@ -7,7 +7,9 @@ SQL > round(count(v.id) * 100.0 / greatest(sum(count(v.id)) OVER (), 1), 0) as percentage FROM vulnerabilities as v FINAL WHERE - repoUrl in (select arrayJoin(repositories) from segments_filtered) AND v.status != 'RESOLVED' + repoUrl in (select arrayJoin(repositories) from segments_filtered) + AND v.repoUrl NOT IN (SELECT url FROM repositories FINAL WHERE archived = true) + AND v.status != 'RESOLVED' {% if defined(repos) %} AND repoUrl IN (SELECT channel FROM repos_to_channels) {% end %} GROUP BY v.packageEcosystem ORDER BY count DESC diff --git a/services/libs/tinybird/pipes/vulnerabilities_by_severity.pipe b/services/libs/tinybird/pipes/vulnerabilities_by_severity.pipe index 2e13d276fe..dd64373e35 100644 --- a/services/libs/tinybird/pipes/vulnerabilities_by_severity.pipe +++ b/services/libs/tinybird/pipes/vulnerabilities_by_severity.pipe @@ -7,7 +7,9 @@ SQL > round(count(v.id) * 100.0 / greatest(sum(count(v.id)) OVER (), 1), 0) as percentage FROM vulnerabilities as v FINAL WHERE - repoUrl in (select arrayJoin(repositories) from segments_filtered) AND v.status != 'RESOLVED' + repoUrl in (select arrayJoin(repositories) from segments_filtered) + AND v.repoUrl NOT IN (SELECT url FROM repositories FINAL WHERE archived = true) + AND v.status != 'RESOLVED' {% if defined(repos) %} AND repoUrl IN (SELECT channel FROM repos_to_channels) {% end %} GROUP BY v.severity ORDER BY count DESC diff --git a/services/libs/tinybird/pipes/vulnerabilities_list.pipe b/services/libs/tinybird/pipes/vulnerabilities_list.pipe index 064e2b927b..1c3edde55c 100644 --- a/services/libs/tinybird/pipes/vulnerabilities_list.pipe +++ b/services/libs/tinybird/pipes/vulnerabilities_list.pipe @@ -6,6 +6,7 @@ SQL > FROM vulnerabilities as v FINAL WHERE repoUrl in (select arrayJoin(repositories) from segments_filtered) + AND v.repoUrl NOT IN (SELECT url FROM repositories FINAL WHERE archived = true) {% if defined(repos) %} AND repoUrl IN (SELECT channel FROM repos_to_channels) {% end %} {% if defined(ecosystem) %} AND v.packageEcosystem = {{ String(ecosystem, required=False) }} @@ -28,6 +29,7 @@ SQL > FROM vulnerabilities as v FINAL WHERE repoUrl in (select arrayJoin(repositories) from segments_filtered) + AND v.repoUrl NOT IN (SELECT url FROM repositories FINAL WHERE archived = true) {% if defined(repos) %} AND repoUrl IN (SELECT channel FROM repos_to_channels) {% end %} {% if defined(ecosystem) %} AND v.packageEcosystem = {{ String(ecosystem, required=False) }} diff --git a/services/libs/tinybird/pipes/vulnerabilities_summary.pipe b/services/libs/tinybird/pipes/vulnerabilities_summary.pipe index d7fb9746af..735a8ad8f4 100644 --- a/services/libs/tinybird/pipes/vulnerabilities_summary.pipe +++ b/services/libs/tinybird/pipes/vulnerabilities_summary.pipe @@ -10,4 +10,5 @@ SQL > FROM vulnerabilities as v FINAL WHERE repoUrl in (select arrayJoin(repositories) from segments_filtered) + AND v.repoUrl NOT IN (SELECT url FROM repositories FINAL WHERE archived = true) {% if defined(repos) %} AND repoUrl IN (SELECT channel FROM repos_to_channels) {% end %} From 1245d2df43818420f5a9050db95d9b12291b3a4c Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Thu, 25 Jun 2026 15:53:15 +0100 Subject: [PATCH 2/2] chore: apply tinybird fmt to insights_projects_populated_copy pipe Signed-off-by: Joana Maia --- .../tinybird/pipes/insights_projects_populated_copy.pipe | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/libs/tinybird/pipes/insights_projects_populated_copy.pipe b/services/libs/tinybird/pipes/insights_projects_populated_copy.pipe index d117b6fccd..afd29b8045 100644 --- a/services/libs/tinybird/pipes/insights_projects_populated_copy.pipe +++ b/services/libs/tinybird/pipes/insights_projects_populated_copy.pipe @@ -189,9 +189,8 @@ SQL > vulnerability_scans vs final ON vs.repoUrl = insights_projects_populated_copy_flatten_projects.repository WHERE - insights_projects_populated_copy_flatten_projects.repository NOT IN ( - SELECT url FROM repositories FINAL WHERE archived = true - ) + insights_projects_populated_copy_flatten_projects.repository + NOT IN (SELECT url FROM repositories FINAL WHERE archived = true) group by insightsProjectId NODE insights_projects_populated_copy_results