Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ repos:
- id: check-ast
- id: check-docstring-first
- repo: https://github.com/psf/black
rev: 25.1.0
rev: 26.1.0
hooks:
- id: black
args: [--quiet]
Expand Down
1 change: 1 addition & 0 deletions database/upgrade/upgrade.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
"""
database upgrade
"""

import os
import subprocess

Expand Down
6 changes: 2 additions & 4 deletions evaluator/processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -263,13 +263,11 @@ async def _mark_system_evaluated(self, cve_cache_count: int, system_platform: Sy

async with conn.cursor() as cur:
await cur.execute(
sql.SQL(
"""UPDATE system_platform
sql.SQL("""UPDATE system_platform
SET last_evaluation = %s,
advisor_evaluated = %s,
cve_count_cache = %s
WHERE id = %s"""
),
WHERE id = %s"""),
(last_evaluation, advisor_evaluated, cve_cache_count, system_platform.id),
)

Expand Down
6 changes: 2 additions & 4 deletions notificator/notificator.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,16 +47,14 @@ async def _create_cve_map(self):
res = {}
async with self.db_pool.acquire() as conn:
async with conn.transaction():
rows = await conn.fetch(
"""SELECT DISTINCT(cm.id), cm.cve, COALESCE(cm.cvss2_score, cm.cvss3_score, 0.0) AS cvss_score,
rows = await conn.fetch("""SELECT DISTINCT(cm.id), cm.cve, COALESCE(cm.cvss2_score, cm.cvss3_score, 0.0) AS cvss_score,
cm.impact_id, cm.exploit_data, cm.advisories_list,
CASE WHEN cm.id IN (SELECT cve_id FROM cve_rule_mapping AS crm
JOIN insights_rule AS ir ON crm.rule_id = ir.id
WHERE active = TRUE) THEN TRUE ELSE FALSE END AS is_rule,
cm.public_date
FROM cve_metadata AS cm
WHERE cm.public_date IS NOT NULL"""
)
WHERE cm.public_date IS NOT NULL""")
for cve_row in rows:
cve = {}
cve["cve"] = cve_row[1]
Expand Down
4 changes: 1 addition & 3 deletions platform_mock/platform_mock.py
Original file line number Diff line number Diff line change
Expand Up @@ -365,9 +365,7 @@ async def get(self, _):
"total_risk": 4,
"hosts_acked_count": 0,
"rating": 0
}""".replace( # noqa: E501
"\n", ""
)
}""".replace("\n", "") # noqa: E501
return JSONResponse(resp)


Expand Down
1 change: 1 addition & 0 deletions platform_mock/traffic_generator.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
"""Kafka traffic generator"""

import argparse
import base64
import json
Expand Down
30 changes: 10 additions & 20 deletions scripts/gabi/cve_stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,7 @@ def main():
print(f"End date: {end_date}")
print("")

top10cves = query(
f"""
top10cves = query(f"""
select cve.cve, cve.cvss3_score, t.cnt
from (
select cve_id, count(*) as cnt
Expand All @@ -41,15 +40,13 @@ def main():
order by cnt desc
limit 10
)t join cve_metadata cve on t.cve_id = cve.id;
"""
)[1:]
""")[1:]
print("Top 10 CVEs with most system hits:")
for cve in top10cves:
print(f"{cve[0]} (CVSS: {cve[1]}): {cve[2]}")
print("")

all_accounts = query(
"""
all_accounts = query("""
select rh_account_id, cnt
from
(
Expand All @@ -62,13 +59,11 @@ def main():
from rh_account
where cve_cache_keepalive is not null
) t2 on t1.rh_account_id = t2.id;
"""
)[1:]
""")[1:]
all_account_cnt = len(all_accounts)
print(f"All actively used accounts: {all_account_cnt}")

accounts = query(
f"""
accounts = query(f"""
select rh_account_id, cnt
from
(
Expand All @@ -83,8 +78,7 @@ def main():
cve_cache_keepalive >= '{start_date}'
) t2 on t1.rh_account_id = t2.id
where t1.cnt >= 10;
"""
)[1:]
""")[1:]
account_cnt = len(accounts)
system_cnt = 0
for acc in accounts:
Expand All @@ -103,8 +97,7 @@ def main():
important_systems_cnt_gt5 = 0
exploited_systems_cnt_gt5 = 0
for idx, account in enumerate(sorted(accounts)):
cnts = query(
f"""
cnts = query(f"""
select impact_id, count(distinct cve_id), count(distinct system_id)
from system_vulnerabilities_active sv join
cve_metadata cve on sv.cve_id = cve.id
Expand All @@ -119,8 +112,7 @@ def main():
)
)
group by impact_id;
"""
)[1:]
""")[1:]
for impact_id, cnt, cnt_sys in cnts:
impact = IMPACTS[impact_id]
cnt = int(cnt)
Expand All @@ -142,8 +134,7 @@ def main():
print(f"ERR: unexpected impact: {impact}")
sys.exit(2)

cnts = query(
f"""
cnts = query(f"""
select count(distinct cve_id), count(distinct system_id)
from system_vulnerabilities_active sv join
cve_metadata cve on sv.cve_id = cve.id
Expand All @@ -157,8 +148,7 @@ def main():
)
)
);
"""
)
""")
cnt = int(cnts[1][0])
cnt_sys = int(cnts[1][1])
# print(f"acc {account[0]} exploited: {cnt}, systems: {cnt_sys}")
Expand Down
7 changes: 3 additions & 4 deletions scripts/gabi/risk_report.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
"""Risk report of affected systems for chosen CVEs."""

import pprint
import re
import sys
Expand Down Expand Up @@ -42,8 +43,7 @@ def main():

cve_sys = {}
for idx in range(PARTITIONS):
cnts = query(
f"""
cnts = query(f"""
select cve.cve, count(distinct system_id)
from system_vulnerabilities_active_{idx} sv join
cve_metadata cve on sv.cve_id = cve.id
Expand All @@ -52,8 +52,7 @@ def main():
sv.first_reported >= '{start_date}' and
sv.first_reported <= '{end_date}'
group by cve.cve;
"""
)[1:]
""")[1:]

for cve, cnt_sys in cnts:
if cve not in cve_sys:
Expand Down
1 change: 1 addition & 0 deletions scripts/generate_insights_archive.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
Generates Insights archive based on VMaaS request.
For testing vulnerability engine only.
"""

import argparse
import datetime
import json
Expand Down
19 changes: 7 additions & 12 deletions taskomatic/jobs/cacheman.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
"""
Periodic account-CVE systems affected cache maintenance
"""

from prometheus_client import Gauge
from psycopg2.extras import execute_values
from psycopg2.sql import SQL
Expand Down Expand Up @@ -78,8 +79,7 @@ def _select_count_affected_granular(account_id):
)

return (
SQL(
"""
SQL("""
SELECT sv.cve_id,
sp.operating_system_id,
sp.group_set_id,
Expand Down Expand Up @@ -111,8 +111,7 @@ def _select_count_affected_granular(account_id):
GROUP BY sv.cve_id,
sp.operating_system_id,
sp.group_set_id
"""
).format(image_patched_cond=image_patched_cond, image_unpatched_cond=image_unpatched_cond),
""").format(image_patched_cond=image_patched_cond, image_unpatched_cond=image_unpatched_cond),
[account_id, account_id],
)

Expand All @@ -123,8 +122,7 @@ def _select_count_unpatched_granular(account_id):
)

return (
SQL(
"""
SQL("""
SELECT vpc.cve_id,
sp.operating_system_id,
sp.group_set_id,
Expand Down Expand Up @@ -153,8 +151,7 @@ def _select_count_unpatched_granular(account_id):
GROUP BY vpc.cve_id,
sp.operating_system_id,
sp.group_set_id
"""
).format(image_unpatched_cond=image_unpatched_cond),
""").format(image_unpatched_cond=image_unpatched_cond),
[account_id, account_id],
)

Expand Down Expand Up @@ -392,13 +389,11 @@ def run():
with METRIC_DURATIONS.labels("prepare").time():
current_cache = {}

cur.execute(
"""SELECT DISTINCT rh_account_id FROM cve_account_granular_cache
cur.execute("""SELECT DISTINCT rh_account_id FROM cve_account_granular_cache
UNION
SELECT DISTINCT rh_account_id FROM rule_account_granular_cache
UNION
SELECT id FROM rh_account WHERE cve_cache_from IS NOT NULL"""
)
SELECT id FROM rh_account WHERE cve_cache_from IS NOT NULL""")
for (account_id,) in cur.fetchall():
current_cache[account_id] = {}

Expand Down
25 changes: 9 additions & 16 deletions taskomatic/jobs/db_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
"""
Periodic DB metrics to Prometheus exporter
"""

from prometheus_client import Gauge

from common.logging import get_logger
Expand Down Expand Up @@ -31,44 +32,36 @@ def run():
cur.execute("""SELECT COUNT(*) FROM inventory.hosts""")
METRIC_CYNDI_SYSTEMS.set(int(cur.fetchone()[0]))

cur.execute(
"""SELECT COUNT(*)
cur.execute("""SELECT COUNT(*)
FROM system_platform sp LEFT JOIN
inventory.hosts ih ON sp.inventory_id = ih.id
WHERE ih.id IS NULL
AND sp.when_deleted IS NULL"""
)
AND sp.when_deleted IS NULL""")
METRIC_SYSTEMS_MISSING_IN_CYNDI.set(int(cur.fetchone()[0]))

cur.execute(
"""SELECT table_schema||'.'||table_name AS key, pg_relation_size('"'||table_schema||'"."'||table_name||'"') AS value
cur.execute("""SELECT table_schema||'.'||table_name AS key, pg_relation_size('"'||table_schema||'"."'||table_name||'"') AS value
FROM information_schema.tables
WHERE table_schema IN ('public', 'inventory', 'repack') AND
table_type = 'BASE TABLE'"""
)
table_type = 'BASE TABLE'""")
for key, value in cur.fetchall():
METRIC_TABLE_SIZE.labels(table=key).set(int(value))

cur.execute(
"""SELECT a.org_id AS account, COUNT(*) AS total_systems
cur.execute("""SELECT a.org_id AS account, COUNT(*) AS total_systems
FROM system_platform sp JOIN rh_account a ON a.id = sp.rh_account_id
GROUP BY a.org_id ORDER BY 2 DESC LIMIT 10"""
)
GROUP BY a.org_id ORDER BY 2 DESC LIMIT 10""")
METRIC_TOP_10_ACCOUNTS_SYSTEMS.clear() # Need to reset because more than 10 labels would be exported when order changes
for account, total_systems in cur.fetchall():
METRIC_TOP_10_ACCOUNTS_SYSTEMS.labels(account=account).set(int(total_systems))

cur.execute(
"""SELECT COUNT(*) FILTER (WHERE t.total_systems >= 1) AS at_least_1_sys,
cur.execute("""SELECT COUNT(*) FILTER (WHERE t.total_systems >= 1) AS at_least_1_sys,
COUNT(*) FILTER (WHERE t.total_systems >= 10) AS at_least_10_sys,
COUNT(*) FILTER (WHERE t.total_systems >= 100) AS at_least_100_sys,
COUNT(*) FILTER (WHERE t.total_systems >= 1000) AS at_least_1000_sys,
COUNT(*) FILTER (WHERE t.total_systems >= 10000) AS at_least_10000_sys,
COUNT(*) FILTER (WHERE t.total_systems >= 100000) AS at_least_100000_sys
FROM (SELECT a.org_id, COUNT(*) AS total_systems
FROM system_platform sp JOIN rh_account a ON a.id = sp.rh_account_id
GROUP BY a.org_id ORDER BY 2 DESC)t"""
)
GROUP BY a.org_id ORDER BY 2 DESC)t""")
at_least_1_sys, at_least_10_sys, at_least_100_sys, at_least_1000_sys, at_least_10000_sys, at_least_100000_sys = cur.fetchone()
METRIC_ACCOUNTS_COUNT.labels(bucket=">= 1 system").set(int(at_least_1_sys))
METRIC_ACCOUNTS_COUNT.labels(bucket=">= 10 systems").set(int(at_least_10_sys))
Expand Down
6 changes: 2 additions & 4 deletions taskomatic/jobs/migrate_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,8 @@ def _system_platform_migrate_rh_account_id(conn, cur):

LOGGER.info("Started _system_platform_migrate_rh_account_id migration.")

cur.execute(
"""SELECT id, rh_account_id from system_platform
WHERE rh_account_id_big IS NULL"""
)
cur.execute("""SELECT id, rh_account_id from system_platform
WHERE rh_account_id_big IS NULL""")

for system_id, rh_account_id in cur.fetchall():
cur.execute(
Expand Down
1 change: 1 addition & 0 deletions taskomatic/jobs/rules_git_sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
"""
Importer for insights rules from insights-content git
"""

import os
import tempfile
from datetime import datetime
Expand Down
1 change: 1 addition & 0 deletions taskomatic/taskomatic.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
"""
Taskomatic service
"""

import asyncio
import importlib
import signal
Expand Down
1 change: 1 addition & 0 deletions tests/common_tests/test_bounded_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
"""
Test suite for bounded executor.
"""

import logging as log

from common.bounded_executor import BoundedExecutor
Expand Down
1 change: 1 addition & 0 deletions tests/common_tests/test_logging.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"""
Test logging.
"""

import logging

import pytest
Expand Down
1 change: 1 addition & 0 deletions tests/common_tests/test_status_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
"""
Test suite for status app.
"""

import asyncio

from common import logging
Expand Down
1 change: 1 addition & 0 deletions tests/common_tests/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
"""
Test suite for utils.
"""

import asyncio
import logging
import sys
Expand Down
1 change: 1 addition & 0 deletions tests/manager_tests/schemas.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"""
Schemas of responses.
"""

from schema import Optional
from schema import Or
from schema import Schema
Expand Down
Loading
Loading