Change url.outcome to url.status

alembic/versions/2025_08_10_2046-5930e70660c5_change_url_outcome_to_url_status.py

@@ -0,0 +1,26 @@
+"""Change URL outcome to URL status
+
+Revision ID: 5930e70660c5
+Revises: 11ece61d7ac2
+Create Date: 2025-08-10 20:46:58.576623
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '5930e70660c5'
+down_revision: Union[str, None] = '11ece61d7ac2'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.alter_column('urls', 'outcome', new_column_name='status')
+
+
+def downgrade() -> None:
+    op.alter_column('urls', 'status', new_column_name='outcome')

src/api/endpoints/annotate/_shared/queries/get_annotation_batch_info.py

@@ -42,7 +42,7 @@ async def run(
         )
 
         common_where_clause = [
-            URL.outcome == URLStatus.PENDING.value,
+            URL.status == URLStatus.PENDING.value,
             LinkBatchURL.batch_id == self.batch_id,
         ]
 

src/api/endpoints/annotate/_shared/queries/get_next_url_for_user_annotation.py

@@ -43,7 +43,7 @@ async def run(self, session: AsyncSession):
 
         query = (
             query
-            .where(URL.outcome == URLStatus.PENDING.value)
+            .where(URL.status == URLStatus.PENDING.value)
             # URL must not have user suggestion
             .where(
                 StatementComposer.user_suggestion_not_exists(self.user_suggestion_model_to_exclude)

src/api/endpoints/annotate/agency/get/queries/next_for_annotation.py

@@ -48,7 +48,7 @@ async def run(
 
         # Must not have confirmed agencies
         query = query.where(
-            URL.outcome == URLStatus.PENDING.value
+            URL.status == URLStatus.PENDING.value
         )
 
 

src/api/endpoints/annotate/all/get/query.py

@@ -39,7 +39,7 @@ async def run(
             query
             .where(
                 and_(
-                    URL.outcome == URLStatus.PENDING.value,
+                    URL.status == URLStatus.PENDING.value,
                     StatementComposer.user_suggestion_not_exists(UserUrlAgencySuggestion),
                     StatementComposer.user_suggestion_not_exists(UserRecordTypeSuggestion),
                     StatementComposer.user_suggestion_not_exists(UserRelevantSuggestion),

src/api/endpoints/collector/manual/query.py

@@ -47,7 +47,7 @@ async def run(self, session: AsyncSession) -> ManualBatchResponseDTO:
                 name=entry.name,
                 description=entry.description,
                 collector_metadata=entry.collector_metadata,
-                outcome=URLStatus.PENDING.value,
+                status=URLStatus.PENDING.value,
                 record_type=entry.record_type.value if entry.record_type is not None else None,
                 source=URLSource.MANUAL
             )

src/api/endpoints/metrics/batches/aggregated/query.py

@@ -43,7 +43,7 @@ def url_column(status: URLStatus, label):
             return sc.count_distinct(
                 case(
                     (
-                        URL.outcome == status.value,
+                        URL.status == status.value,
                         URL.id
                     )
                 ),

src/api/endpoints/metrics/batches/breakdown/query.py

@@ -36,7 +36,7 @@ def url_column(status: URLStatus, label):
             return sc.count_distinct(
                 case(
                     (
-                        URL.outcome == status.value,
+                        URL.status == status.value,
                         URL.id
                     )
                 ),

src/api/endpoints/review/approve/query_/core.py

@@ -95,7 +95,7 @@ async def run(self, session: AsyncSession) -> None:
 
         # If it does, do nothing
 
-        url.outcome = URLStatus.VALIDATED.value
+        url.status = URLStatus.VALIDATED.value
 
         update_if_not_none(url, "name", self.approval_info.name, required=True)
         update_if_not_none(url, "description", self.approval_info.description, required=False)

src/api/endpoints/review/next/query.py

@@ -93,7 +93,7 @@ def _build_base_query(
         query = (
             query.where(
                 and_(
-                    URL.outcome == URLStatus.PENDING.value,
+                    URL.status == URLStatus.PENDING.value,
                     *where_exist_clauses
                 )
             )
@@ -189,7 +189,7 @@ async def get_count_ready_query(self):
             )
             .where(
                 LinkBatchURL.batch_id == self.batch_id,
-                URL.outcome == URLStatus.PENDING.value,
+                URL.status == URLStatus.PENDING.value,
                 *self._get_where_exist_clauses(
                     builder.query
                 )
@@ -209,7 +209,7 @@ async def get_count_reviewed_query(self):
             .join(LinkBatchURL)
             .outerjoin(URL, URL.id == LinkBatchURL.url_id)
             .where(
-                URL.outcome.in_(
+                URL.status.in_(
                     [
                         URLStatus.VALIDATED.value,
                         URLStatus.NOT_RELEVANT.value,

src/api/endpoints/review/reject/query.py

@@ -35,11 +35,11 @@ async def run(self, session) -> None:
 
         match self.rejection_reason:
             case RejectionReason.INDIVIDUAL_RECORD:
-                url.outcome = URLStatus.INDIVIDUAL_RECORD.value
+                url.status = URLStatus.INDIVIDUAL_RECORD.value
             case RejectionReason.BROKEN_PAGE_404:
-                url.outcome = URLStatus.NOT_FOUND.value
+                url.status = URLStatus.NOT_FOUND.value
             case RejectionReason.NOT_RELEVANT:
-                url.outcome = URLStatus.NOT_RELEVANT.value
+                url.status = URLStatus.NOT_RELEVANT.value
             case _:
                 raise HTTPException(
                     status_code=HTTP_400_BAD_REQUEST,

src/api/endpoints/task/by_id/query.py

@@ -43,7 +43,7 @@ async def run(self, session: AsyncSession) -> TaskInfo:
                 batch_id=url.batch.id,
                 url=url.url,
                 collector_metadata=url.collector_metadata,
-                outcome=URLStatus(url.outcome),
+                status=URLStatus(url.status),
                 updated_at=url.updated_at
             )
             url_infos.append(url_info)

src/api/endpoints/url/get/query.py

@@ -51,7 +51,7 @@ async def run(self, session: AsyncSession) -> GetURLsResponseInfo:
                     id=result.id,
                     batch_id=result.batch.id if result.batch is not None else None,
                     url=result.url,
-                    status=URLStatus(result.outcome),
+                    status=URLStatus(result.status),
                     collector_metadata=result.collector_metadata,
                     updated_at=result.updated_at,
                     created_at=result.created_at,

src/collectors/queries/insert/url.py

@@ -18,7 +18,7 @@ async def run(self, session: AsyncSession) -> int:
         url_entry = URL(
             url=self.url_info.url,
             collector_metadata=self.url_info.collector_metadata,
-            outcome=self.url_info.outcome.value,
+            status=self.url_info.status.value,
             source=self.url_info.source
         )
         if self.url_info.created_at is not None:

src/core/tasks/scheduled/impl/huggingface/queries/check/requester.py

@@ -35,7 +35,7 @@ async def has_valid_urls(self, last_upload_at: datetime | None) -> bool:
                 URL.id == URLCompressedHTML.url_id
             )
             .where(
-                URL.outcome.in_(
+                URL.status.in_(
                     [
                         URLStatus.VALIDATED,
                         URLStatus.NOT_RELEVANT.value,

src/core/tasks/scheduled/impl/huggingface/queries/get/core.py

@@ -26,7 +26,7 @@ async def run(self, session: AsyncSession) -> list[GetForLoadingToHuggingFaceOut
             select(
                 URL.id.label(label_url_id),
                 URL.url.label(label_url),
-                URL.outcome.label(label_url_status),
+                URL.status.label(label_url_status),
                 URL.record_type.label(label_record_type_fine),
                 URLCompressedHTML.compressed_html.label(label_html)
             )
@@ -35,7 +35,7 @@ async def run(self, session: AsyncSession) -> list[GetForLoadingToHuggingFaceOut
                 URL.id == URLCompressedHTML.url_id
             )
             .where(
-                URL.outcome.in_([
+                URL.status.in_([
                     URLStatus.VALIDATED,
                     URLStatus.NOT_RELEVANT,
                     URLStatus.SUBMITTED

src/core/tasks/scheduled/impl/sync/data_sources/queries/upsert/core.py

@@ -9,6 +9,8 @@
 from src.core.tasks.scheduled.impl.sync.data_sources.queries.upsert.param_manager import \
     UpsertURLsFromDataSourcesParamManager
 from src.core.tasks.scheduled.impl.sync.data_sources.queries.upsert.requester import UpsertURLsFromDataSourcesDBRequester
+from src.core.tasks.scheduled.impl.sync.data_sources.queries.upsert.url.insert.params import \
+    InsertURLForDataSourcesSyncParams
 from src.core.tasks.scheduled.impl.sync.data_sources.queries.upsert.url.lookup.response import \
     LookupURLForDataSourcesSyncResponse
 from src.db.dtos.url.mapping import URLMapping
@@ -84,7 +86,7 @@ async def _add_new_data_sources(self, url_mappings: list[URLMapping]):
         await self.requester.add_new_data_sources(url_ds_insert_params)
 
     async def _add_new_urls(self, urls: list[str]):
-        url_insert_params = self.param_manager.add_new_urls(urls)
+        url_insert_params: list[InsertURLForDataSourcesSyncParams] = self.param_manager.add_new_urls(urls)
         url_mappings = await self.requester.add_new_urls(url_insert_params)
         return url_mappings
 

src/core/tasks/scheduled/impl/sync/data_sources/queries/upsert/helpers/convert.py

@@ -41,7 +41,7 @@ def convert_to_url_update_params(
         id=url_id,
         name=sync_info.name,
         description=sync_info.description,
-        outcome=convert_to_source_collector_url_status(
+        status=convert_to_source_collector_url_status(
             ds_url_status=sync_info.url_status,
             ds_approval_status=sync_info.approval_status
         ),
@@ -56,7 +56,7 @@ def convert_to_url_insert_params(
         url=url,
         name=sync_info.name,
         description=sync_info.description,
-        outcome=convert_to_source_collector_url_status(
+        status=convert_to_source_collector_url_status(
             ds_url_status=sync_info.url_status,
             ds_approval_status=sync_info.approval_status
         ),

src/core/tasks/scheduled/impl/sync/data_sources/queries/upsert/url/insert/params.py

@@ -9,7 +9,7 @@ class InsertURLForDataSourcesSyncParams(BulkInsertableModel):
     url: str
     name: str
     description: str | None
-    outcome: URLStatus
+    status: URLStatus
     record_type: RecordType
     source: URLSource = URLSource.DATA_SOURCES
 

src/core/tasks/scheduled/impl/sync/data_sources/queries/upsert/url/update/params.py

@@ -17,5 +17,5 @@ def sa_model(cls) -> type[URL]:
     id: int
     name: str
     description: str | None
-    outcome: URLStatus
+    status: URLStatus
     record_type: RecordType

src/core/tasks/url/operators/agency_identification/queries/get_pending_urls_without_agency_suggestions.py

@@ -21,7 +21,7 @@ async def run(self, session: AsyncSession) -> list[AgencyIdentificationTDO]:
                 Batch.strategy
             )
             .select_from(URL)
-            .where(URL.outcome == URLStatus.PENDING.value)
+            .where(URL.status == URLStatus.PENDING.value)
             .outerjoin(LinkBatchURL)
             .outerjoin(Batch)
         )

src/core/tasks/url/operators/agency_identification/queries/has_urls_without_agency_suggestions.py

@@ -17,7 +17,7 @@ async def run(
             select(
                 URL.id
             ).where(
-                URL.outcome == URLStatus.PENDING.value
+                URL.status == URLStatus.PENDING.value
             )
         )
 

src/core/tasks/url/operators/auto_relevant/queries/get_tdos.py

@@ -29,7 +29,7 @@ async def run(self, session: AsyncSession) -> list[URLRelevantTDO]:
             )
             .join(URLCompressedHTML)
             .where(
-                URL.outcome == URLStatus.PENDING.value,
+                URL.status == URLStatus.PENDING.value,
             )
         )
         query = StatementComposer.exclude_urls_with_extant_model(

src/core/tasks/url/operators/html/queries/get.py

@@ -21,7 +21,7 @@ async def run(self, session: AsyncSession) -> list[URLInfo]:
                 batch_id=url.batch.id if url.batch is not None else None,
                 url=url.url,
                 collector_metadata=url.collector_metadata,
-                outcome=url.outcome,
+                status=url.status,
                 created_at=url.created_at,
                 updated_at=url.updated_at,
                 name=url.name

src/core/tasks/url/operators/submit_approved/queries/get.py

@@ -29,7 +29,7 @@ async def _process_results(self, urls):
     async def _build_query():
         query = (
             select(URL)
-            .where(URL.outcome == URLStatus.VALIDATED.value)
+            .where(URL.status == URLStatus.VALIDATED.value)
             .options(
                 selectinload(URL.optional_data_source_metadata),
                 selectinload(URL.confirmed_agencies),

src/core/tasks/url/operators/submit_approved/queries/has_validated.py

@@ -11,7 +11,7 @@ class HasValidatedURLsQueryBuilder(QueryBuilderBase):
     async def run(self, session: AsyncSession) -> bool:
         query = (
             select(URL)
-            .where(URL.outcome == URLStatus.VALIDATED.value)
+            .where(URL.status == URLStatus.VALIDATED.value)
         )
         urls = await session.execute(query)
         urls = urls.scalars().all()

src/core/tasks/url/operators/submit_approved/queries/mark_submitted.py

@@ -23,7 +23,7 @@ async def run(self, session: AsyncSession):
                 update(URL)
                 .where(URL.id == url_id)
                 .values(
-                    outcome=URLStatus.SUBMITTED.value
+                    status=URLStatus.SUBMITTED.value
                 )
             )