Oonirun v2 1

ooniapi/common/src/common/alembic/versions/b860eb79750f_add_targets_name_and_inputs_extra_.py

@@ -0,0 +1,36 @@
+"""Add targets_name and inputs_extra columns to oonirun_nettest
+
+Revision ID: b860eb79750f
+Revises: 8e7ecea5c2f5
+Create Date: 2025-05-21 15:44:32.959349
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "b860eb79750f"
+down_revision: Union[str, None] = "8e7ecea5c2f5"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "oonirun_nettest", sa.Column("targets_name", sa.String(), nullable=True)
+    )
+    op.add_column(
+        "oonirun_nettest", sa.Column("inputs_extra", sa.ARRAY(sa.JSON()), nullable=True)
+    )
+    op.drop_column("oonirun_nettest", "backend_options")
+
+
+def downgrade() -> None:
+    op.drop_column("oonirun_nettest", "targets_name")
+    op.drop_column("oonirun_nettest", "inputs_extra")
+
+    op.add_column("oonirun_nettest", sa.Column("backend_options", sa.ARRAY(sa.JSON()), nullable=True))

ooniapi/common/src/common/dependencies.py

@@ -1,8 +1,13 @@
 from functools import lru_cache
-from typing import Annotated
+from typing import Annotated, TypeAlias
+
+from clickhouse_driver import Client as Clickhouse
 
 from fastapi import Depends
 from fastapi import HTTPException, Header
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker, Session
+
 from .auth import get_client_token
 from .config import Settings
 
@@ -11,6 +16,7 @@
 def get_settings() -> Settings:
     return Settings()
 
+SettingsDep: TypeAlias = Annotated[Settings, Depends(get_settings)]
 
 def role_required(roles: list[str]):
     """Wrapped function requiring user to be logged in and have the right role."""
@@ -38,3 +44,28 @@ async def verify_jwt(
         return tok
 
     return verify_jwt
+
+
+def get_clickhouse_session(settings: SettingsDep):
+    db = Clickhouse.from_url(settings.clickhouse_url)
+    try:
+        yield db
+    finally:
+        db.disconnect()
+
+
+ClickhouseDep = Annotated[Clickhouse, Depends(get_clickhouse_session)]
+
+
+def get_postgresql_session(settings: SettingsDep):
+    engine = create_engine(settings.postgresql_url)
+    SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()
+
+
+PostgresDep = Annotated[Session, Depends(get_postgresql_session)]

ooniapi/common/src/common/postgresql.py

@@ -9,4 +9,5 @@ class Base(DeclarativeBase):
         Dict[str, Any]: sa.JSON,
         List[str]: sa.JSON,
         Dict[str, str]: sa.JSON,
+        List[Dict[str, Any]] : sa.ARRAY(sa.JSON())
     }

ooniapi/services/ooniprobe/src/ooniprobe/prio.py → ooniapi/common/src/common/prio.py

@@ -1,3 +1,4 @@
+
 """
 OONI Probe Services API - reactive URL prioritization
 
@@ -32,8 +33,8 @@
 from fastapi import Depends
 from pydantic import BaseModel
 
-from .common.clickhouse_utils import query_click
-from .common.metrics import timer
+from .clickhouse_utils import query_click
+from .metrics import timer
 from .dependencies import ClickhouseDep
 
 log = logging.getLogger(__name__)

ooniapi/services/ooniauth/pyproject.toml

@@ -24,6 +24,7 @@ classifiers = [
 ]
 dependencies = [
   "fastapi ~= 0.108.0",
+  "clickhouse-driver ~= 0.2.6",
   "sqlalchemy ~= 2.0.27",
   "ujson ~= 5.9.0",
   "python-dateutil ~= 2.8.2",

ooniapi/services/oonifindings/src/oonifindings/dependencies.py

@@ -1,30 +0,0 @@
-from typing import Annotated
-
-from clickhouse_driver import Client as Clickhouse
-
-from fastapi import Depends
-
-from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker
-
-from .common.config import Settings
-from .common.dependencies import get_settings
-
-
-def get_postgresql_session(settings: Annotated[Settings, Depends(get_settings)]):
-    engine = create_engine(settings.postgresql_url)
-    SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
-
-    db = SessionLocal()
-    try:
-        yield db
-    finally:
-        db.close()
-
-
-def get_clickhouse_session(settings: Annotated[Settings, Depends(get_settings)]):
-    db = Clickhouse.from_url(settings.clickhouse_url)
-    try:
-        yield db
-    finally:
-        db.disconnect()

ooniapi/services/oonifindings/src/oonifindings/main.py

@@ -12,7 +12,7 @@
 from .routers import v1
 
 
-from .dependencies import get_settings, get_postgresql_session
+from .common.dependencies import get_settings, get_postgresql_session
 from .common.version import get_build_label, get_pkg_version
 from .common.metrics import mount_metrics
 
@@ -40,7 +40,7 @@ async def lifespan(app: FastAPI):
 
 app.add_middleware(
     CORSMiddleware,
-    allow_origin_regex="^https://[-A-Za-z0-9]+(\.test)?\.ooni\.(org|io)$",
+    allow_origin_regex=r"^https://[-A-Za-z0-9]+(\.test)?\.ooni\.(org|io)$",
     allow_credentials=True,
     allow_methods=["*"],
     allow_headers=["*"],

ooniapi/services/oonifindings/src/oonifindings/routers/v1.py

@@ -23,7 +23,7 @@
     get_client_role,
 )
 from ..common.utils import setnocacheresponse, generate_random_intuid
-from ..dependencies import get_postgresql_session
+from ..common.dependencies import SettingsDep, PostgresDep
 
 log = logging.getLogger(__name__)
 
@@ -128,6 +128,8 @@ class OONIFindingIncidents(BaseModel):
 )
 def list_oonifindings(
     response: Response,
+    db: PostgresDep,
+    settings: SettingsDep,
     only_mine: Annotated[bool, Query(description="show only owned items")] = False,
     theme: Annotated[
         list[str] | None,
@@ -146,8 +148,6 @@ def list_oonifindings(
         Query(description="The domain to filter by"),
     ] = None,
     authorization: str = Header("authorization"),
-    db=Depends(get_postgresql_session),
-    settings=Depends(get_settings),
 ):
     """
     Search and list incidents
@@ -227,9 +227,9 @@ def list_oonifindings(
 def get_oonifinding_by_id(
     finding_id: str,
     response: Response,
+    db: PostgresDep,
+    settings: SettingsDep,
     authorization: str = Header("authorization"),
-    db=Depends(get_postgresql_session),
-    settings=Depends(get_settings),
 ):
     """
     Returns an incident
@@ -323,10 +323,10 @@ def generate_finding_slug(create_time: datetime, title: str):
 def create_oonifinding(
     create_request: OONIFindingCreateUpdate,
     response: Response,
+    db: PostgresDep,
+    settings: SettingsDep,
     authorization: str = Header("authorization"),
     token=Depends(role_required(["admin"])),
-    db=Depends(get_postgresql_session),
-    settings=Depends(get_settings),
 ):
     """
     Create an incident
@@ -388,7 +388,7 @@ def create_oonifinding(
 def update_oonifinding(
     update_request: OONIFindingCreateUpdate,
     response: Response,
-    db=Depends(get_postgresql_session),
+    db: PostgresDep,
     token=Depends(role_required(["admin", "user"])),
 ):
     """
@@ -457,8 +457,8 @@ def update_oonifinding(
 def delete_oonifinding(
     delete_request: OONIFindingWithMail,
     response: Response,
+    db: PostgresDep,
     token=Depends(role_required(["admin", "user"])),
-    db=Depends(get_postgresql_session),
 ):
     """
     Delete an incident
@@ -503,7 +503,7 @@ def update_oonifinding_publish_status(
     action: str,
     publish_request: OONIFindingId,
     response: Response,
-    db=Depends(get_postgresql_session),
+    db: PostgresDep,
 ):
     """
     Publish/Unpublish an incident.

ooniapi/services/oonimeasurements/src/oonimeasurements/main.py

@@ -17,8 +17,7 @@
     aggregate_analysis,
 )
 
-from .dependencies import get_clickhouse_session
-from .common.dependencies import get_settings
+from .common.dependencies import get_settings, get_clickhouse_session
 from .common.version import get_build_label, get_pkg_version
 from .common.clickhouse_utils import query_click
 from .common.metrics import mount_metrics

ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/aggregate_analysis.py

@@ -20,7 +20,6 @@
 
 import logging
 
-from fastapi import APIRouter
 
 router = APIRouter()
 

ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/aggregate_observations.py

@@ -5,7 +5,7 @@
 from pydantic import BaseModel
 
 
-from ...dependencies import get_clickhouse_session
+from ...common.dependencies import get_clickhouse_session
 from .utils import (
     SinceUntil,
     get_measurement_start_day_agg,

ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/list_analysis.py

@@ -12,14 +12,12 @@
 from .utils import (
     SinceUntil,
     parse_probe_asn_to_int,
-    test_name_to_group,
     utc_30_days_ago,
     utc_today,
 )
 
 log = logging.getLogger(__name__)
 
-from fastapi import APIRouter
 
 router = APIRouter()
 

ooniapi/services/oonimeasurements/src/oonimeasurements/routers/data/list_observations.py

@@ -7,8 +7,7 @@
 from pydantic import BaseModel
 from typing_extensions import Annotated
 
-from ...common.dependencies import get_settings
-from ...dependencies import get_clickhouse_session
+from ...common.dependencies import get_settings, get_clickhouse_session
 from .utils import parse_probe_asn_to_int
 
 from fastapi import APIRouter

ooniapi/services/oonimeasurements/src/oonimeasurements/routers/v1/aggregation.py

@@ -18,7 +18,7 @@
 
 from oonimeasurements.common.clickhouse_utils import query_click, query_click_one_row
 from oonimeasurements.common.utils import jerror, commasplit, convert_to_csv
-from ...dependencies import get_clickhouse_session
+from oonimeasurements.common.dependencies import get_clickhouse_session
 from ...common.routers import BaseModel
 
 router = APIRouter()

ooniapi/services/oonimeasurements/src/oonimeasurements/routers/v1/measurements.py

@@ -46,7 +46,7 @@
 from ...common.routers import BaseModel
 from ...common.utils import setcacheresponse, commasplit, setnocacheresponse
 from ...common.clickhouse_utils import query_click, query_click_one_row
-from ...dependencies import get_clickhouse_session
+from ...common.dependencies import get_clickhouse_session
 
 log = logging.getLogger(__name__)
 

ooniapi/services/oonimeasurements/tests/conftest.py

@@ -9,7 +9,7 @@
 from clickhouse_driver import Client as ClickhouseClient
 
 from oonimeasurements.common.config import Settings
-from oonimeasurements.dependencies import get_settings
+from oonimeasurements.common.dependencies import get_settings
 from oonimeasurements.main import app
 
 THIS_DIR = Path(__file__).parent.resolve()

ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py

@@ -1,23 +1,20 @@
+import io
 from typing import Annotated, TypeAlias, Any, Dict
 from datetime import datetime
 import time
 from pathlib import Path
 
+import boto3
+import ujson
+import geoip2.database
 from fastapi import Depends
 
-import geoip2.database
+from mypy_boto3_s3 import S3Client
 
-import ujson
 from pydantic import BaseModel, Field
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker, Session
 
-from clickhouse_driver import Client as Clickhouse
-import io
-
-import boto3
-from mypy_boto3_s3 import S3Client
-
 from .common.config import Settings
 from .common.dependencies import get_settings
 
@@ -54,17 +51,6 @@ def get_asn_reader(settings: SettingsDep):
 ASNReaderDep = Annotated[geoip2.database.Reader, Depends(get_asn_reader)]
 
 
-def get_clickhouse_session(settings: SettingsDep):
-    db = Clickhouse.from_url(settings.clickhouse_url)
-    try:
-        yield db
-    finally:
-        db.disconnect()
-
-
-ClickhouseDep = Annotated[Clickhouse, Depends(get_clickhouse_session)]
-
-
 def get_s3_client() -> S3Client:
     s3 = boto3.client("s3")
     return s3