Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/api/endpoints/batch/duplicates/query.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ async def run(self, session: AsyncSession) -> list[DuplicateInfo]:
final_results.append(
DuplicateInfo(
source_url=result.source_url,
duplicate_batch_id=result.duplicate_batch_id,
batch_id=result.duplicate_batch_id,
duplicate_metadata=result.duplicate_batch_parameters,
original_batch_id=result.original_batch_id,
original_metadata=result.original_batch_parameters,
Expand Down
14 changes: 7 additions & 7 deletions src/api/endpoints/collector/routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,17 @@
from src.api.endpoints.collector.dtos.collector_start import CollectorStartInfo
from src.api.endpoints.collector.dtos.manual_batch.post import ManualBatchInputDTO
from src.api.endpoints.collector.dtos.manual_batch.response import ManualBatchResponseDTO
from src.collectors.source_collectors.auto_googler.dtos.input import AutoGooglerInputDTO
from src.collectors.source_collectors.common_crawler.input import CommonCrawlerInputDTO
from src.collectors.source_collectors.example.dtos.input import ExampleInputDTO
from src.collectors.impl.auto_googler.dtos.input import AutoGooglerInputDTO
from src.collectors.impl.common_crawler.input import CommonCrawlerInputDTO
from src.collectors.impl.example.dtos.input import ExampleInputDTO
from src.collectors.enums import CollectorType
from src.core.core import AsyncCore
from src.security.manager import get_access_info
from src.security.dtos.access_info import AccessInfo
from src.collectors.source_collectors.ckan.dtos.input import CKANInputDTO
from src.collectors.source_collectors.muckrock.collectors.all_foia.dto import MuckrockAllFOIARequestsCollectorInputDTO
from src.collectors.source_collectors.muckrock.collectors.county.dto import MuckrockCountySearchCollectorInputDTO
from src.collectors.source_collectors.muckrock.collectors.simple.dto import MuckrockSimpleSearchCollectorInputDTO
from src.collectors.impl.ckan.dtos.input import CKANInputDTO
from src.collectors.impl.muckrock.collectors.all_foia.dto import MuckrockAllFOIARequestsCollectorInputDTO
from src.collectors.impl.muckrock.collectors.county.dto import MuckrockCountySearchCollectorInputDTO
from src.collectors.impl.muckrock.collectors.simple.dto import MuckrockSimpleSearchCollectorInputDTO

collector_router = APIRouter(
prefix="/collector",
Expand Down
2 changes: 1 addition & 1 deletion src/api/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from src.api.endpoints.task.routes import task_router
from src.api.endpoints.url.routes import url_router
from src.collectors.manager import AsyncCollectorManager
from src.collectors.source_collectors.muckrock.api_interface.core import MuckrockAPIInterface
from src.collectors.impl.muckrock.api_interface.core import MuckrockAPIInterface
from src.core.core import AsyncCore
from src.core.logger import AsyncCoreLogger
from src.core.env_var_manager import EnvVarManager
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from src.collectors.source_collectors.auto_googler.dtos.query_results import GoogleSearchQueryResultsInnerDTO
from src.collectors.source_collectors.auto_googler.searcher import GoogleSearcher
from src.collectors.source_collectors.auto_googler.dtos.config import SearchConfig
from src.collectors.impl.auto_googler.dtos.query_results import GoogleSearchQueryResultsInnerDTO

Check warning on line 1 in src/collectors/impl/auto_googler/auto_googler.py

View workflow job for this annotation

GitHub Actions / flake8

[flake8] src/collectors/impl/auto_googler/auto_googler.py#L1 <100>

Missing docstring in public module
Raw output
./src/collectors/impl/auto_googler/auto_googler.py:1:1: D100 Missing docstring in public module
from src.collectors.impl.auto_googler.searcher import GoogleSearcher
from src.collectors.impl.auto_googler.dtos.config import SearchConfig


class AutoGoogler:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@

from src.collectors.source_collectors.base import AsyncCollectorBase
from src.collectors.impl.base import AsyncCollectorBase
from src.collectors.enums import CollectorType
from src.core.env_var_manager import EnvVarManager
from src.core.preprocessors.autogoogler import AutoGooglerPreprocessor
from src.collectors.source_collectors.auto_googler.auto_googler import AutoGoogler
from src.collectors.source_collectors.auto_googler.dtos.output import AutoGooglerInnerOutputDTO
from src.collectors.source_collectors.auto_googler.dtos.input import AutoGooglerInputDTO
from src.collectors.source_collectors.auto_googler.searcher import GoogleSearcher
from src.collectors.source_collectors.auto_googler.dtos.config import SearchConfig
from src.collectors.impl.auto_googler.auto_googler import AutoGoogler
from src.collectors.impl.auto_googler.dtos.output import AutoGooglerInnerOutputDTO
from src.collectors.impl.auto_googler.dtos.input import AutoGooglerInputDTO
from src.collectors.impl.auto_googler.searcher import GoogleSearcher
from src.collectors.impl.auto_googler.dtos.config import SearchConfig
from src.util.helper_functions import base_model_list_dump


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
import aiohttp
from googleapiclient.errors import HttpError

from src.collectors.source_collectors.auto_googler.dtos.query_results import GoogleSearchQueryResultsInnerDTO
from src.collectors.source_collectors.auto_googler.exceptions import QuotaExceededError
from src.collectors.impl.auto_googler.dtos.query_results import GoogleSearchQueryResultsInnerDTO
from src.collectors.impl.auto_googler.exceptions import QuotaExceededError


class GoogleSearcher:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from src.core.function_trigger import FunctionTrigger
from src.core.enums import BatchStatus
from src.core.preprocessors.base import PreprocessorBase
from src.db.models.instantiations.url.core.pydantic.info import URLInfo


class AsyncCollectorBase(ABC):
Expand Down Expand Up @@ -73,8 +74,8 @@ async def handle_error(self, e: Exception) -> None:

async def process(self) -> None:
await self.log("Processing collector...")
preprocessor = self.preprocessor()
url_infos = preprocessor.preprocess(self.data)
preprocessor: PreprocessorBase = self.preprocessor()
url_infos: list[URLInfo] = preprocessor.preprocess(self.data)
await self.log(f"URLs processed: {len(url_infos)}")

await self.log("Inserting URLs...")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
from pydantic import BaseModel

from src.collectors.source_collectors.base import AsyncCollectorBase
from src.collectors.impl.base import AsyncCollectorBase
from src.collectors.enums import CollectorType
from src.core.preprocessors.ckan import CKANPreprocessor
from src.collectors.source_collectors.ckan.dtos.input import CKANInputDTO
from src.collectors.source_collectors.ckan.scraper_toolkit.search_funcs.group import ckan_group_package_search
from src.collectors.source_collectors.ckan.scraper_toolkit.search_funcs.organization import ckan_package_search_from_organization
from src.collectors.source_collectors.ckan.scraper_toolkit.search_funcs.package import ckan_package_search
from src.collectors.source_collectors.ckan.scraper_toolkit.search import perform_search, get_flat_list, deduplicate_entries, \
from src.collectors.impl.ckan.dtos.input import CKANInputDTO
from src.collectors.impl.ckan.scraper_toolkit.search_funcs.group import ckan_group_package_search
from src.collectors.impl.ckan.scraper_toolkit.search_funcs.organization import ckan_package_search_from_organization
from src.collectors.impl.ckan.scraper_toolkit.search_funcs.package import ckan_package_search
from src.collectors.impl.ckan.scraper_toolkit.search import perform_search, get_flat_list, deduplicate_entries, \
get_collections, filter_result, parse_result
from src.util.helper_functions import base_model_list_dump

Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from pydantic import BaseModel, Field

from src.collectors.source_collectors.ckan.dtos.search.group_and_organization import GroupAndOrganizationSearchDTO
from src.collectors.source_collectors.ckan.dtos.search.package import CKANPackageSearchDTO
from src.collectors.impl.ckan.dtos.search.group_and_organization import GroupAndOrganizationSearchDTO
from src.collectors.impl.ckan.dtos.search.package import CKANPackageSearchDTO


class CKANInputDTO(BaseModel):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

from pydantic import BaseModel, Field

from src.collectors.source_collectors.ckan.dtos.search._helpers import url_field
from src.collectors.impl.ckan.dtos.search._helpers import url_field


class GroupAndOrganizationSearchDTO(BaseModel):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

from pydantic import BaseModel, Field

from src.collectors.source_collectors.ckan.dtos.search._helpers import url_field
from src.collectors.impl.ckan.dtos.search._helpers import url_field


class CKANPackageSearchDTO(BaseModel):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import aiohttp
from aiohttp import ContentTypeError

from src.collectors.source_collectors.ckan.exceptions import CKANAPIError
from src.collectors.impl.ckan.exceptions import CKANAPIError


class CKANAPIInterface:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
from from_root import from_root
from tqdm import tqdm

from src.collectors.source_collectors.ckan.scraper_toolkit.search_funcs.collection import ckan_collection_search
from src.collectors.source_collectors.ckan.dtos.package import Package
from src.collectors.source_collectors.ckan.constants import CKAN_DATA_TYPES, CKAN_TYPE_CONVERSION_MAPPING
from src.collectors.impl.ckan.scraper_toolkit.search_funcs.collection import ckan_collection_search
from src.collectors.impl.ckan.dtos.package import Package
from src.collectors.impl.ckan.constants import CKAN_DATA_TYPES, CKAN_TYPE_CONVERSION_MAPPING

p = from_root(".pydocstyle").parent
sys.path.insert(1, str(p))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import aiohttp
from bs4 import ResultSet, Tag, BeautifulSoup

from src.collectors.source_collectors.ckan.dtos.package import Package
from src.collectors.impl.ckan.dtos.package import Package


async def ckan_collection_search(base_url: str, collection_id: str) -> list[Package]:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import sys
from typing import Optional, Any

from src.collectors.source_collectors.ckan.scraper_toolkit._api_interface import CKANAPIInterface
from src.collectors.impl.ckan.scraper_toolkit._api_interface import CKANAPIInterface


async def ckan_group_package_search(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from typing import Any

from src.collectors.source_collectors.ckan.scraper_toolkit._api_interface import CKANAPIInterface
from src.collectors.source_collectors.ckan.scraper_toolkit.search_funcs.package import ckan_package_search
from src.collectors.impl.ckan.scraper_toolkit._api_interface import CKANAPIInterface
from src.collectors.impl.ckan.scraper_toolkit.search_funcs.package import ckan_package_search


async def ckan_package_search_from_organization(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import sys
from typing import Optional, Any

from src.collectors.source_collectors.ckan.scraper_toolkit._api_interface import CKANAPIInterface
from src.collectors.impl.ckan.scraper_toolkit._api_interface import CKANAPIInterface


async def ckan_package_search(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
from src.collectors.source_collectors.base import AsyncCollectorBase
from src.collectors.impl.base import AsyncCollectorBase

Check warning on line 1 in src/collectors/impl/common_crawler/collector.py

View workflow job for this annotation

GitHub Actions / flake8

[flake8] src/collectors/impl/common_crawler/collector.py#L1 <100>

Missing docstring in public module
Raw output
./src/collectors/impl/common_crawler/collector.py:1:1: D100 Missing docstring in public module
from src.collectors.enums import CollectorType
from src.core.preprocessors.common_crawler import CommonCrawlerPreprocessor
from src.collectors.source_collectors.common_crawler.crawler import CommonCrawler
from src.collectors.source_collectors.common_crawler.input import CommonCrawlerInputDTO
from src.collectors.impl.common_crawler.crawler import CommonCrawler
from src.collectors.impl.common_crawler.input import CommonCrawlerInputDTO


class CommonCrawlerCollector(AsyncCollectorBase):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

import aiohttp

from src.collectors.source_collectors.common_crawler.utils import URLWithParameters
from src.collectors.impl.common_crawler.utils import URLWithParameters

async def async_make_request(
search_url: 'URLWithParameters'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
"""
import asyncio

from src.collectors.source_collectors.base import AsyncCollectorBase
from src.collectors.source_collectors.example.dtos.input import ExampleInputDTO
from src.collectors.source_collectors.example.dtos.output import ExampleOutputDTO
from src.collectors.impl.base import AsyncCollectorBase
from src.collectors.impl.example.dtos.input import ExampleInputDTO
from src.collectors.impl.example.dtos.output import ExampleOutputDTO
from src.collectors.enums import CollectorType
from src.core.preprocessors.example import ExamplePreprocessor

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
import requests
from aiohttp import ClientSession

from src.collectors.source_collectors.muckrock.api_interface.lookup_response import AgencyLookupResponse
from src.collectors.source_collectors.muckrock.enums import AgencyLookupResponseType
from src.collectors.impl.muckrock.api_interface.lookup_response import AgencyLookupResponse
from src.collectors.impl.muckrock.enums import AgencyLookupResponseType


class MuckrockAPIInterface:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

from pydantic import BaseModel

from src.collectors.source_collectors.muckrock.enums import AgencyLookupResponseType
from src.collectors.impl.muckrock.enums import AgencyLookupResponseType


class AgencyLookupResponse(BaseModel):
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
from src.collectors.enums import CollectorType
from src.collectors.source_collectors.base import AsyncCollectorBase
from src.collectors.source_collectors.muckrock.collectors.all_foia.dto import MuckrockAllFOIARequestsCollectorInputDTO
from src.collectors.source_collectors.muckrock.fetchers.foia.core import FOIAFetcher
from src.collectors.source_collectors.muckrock.exceptions import MuckrockNoMoreDataError
from src.collectors.impl.base import AsyncCollectorBase
from src.collectors.impl.muckrock.collectors.all_foia.dto import MuckrockAllFOIARequestsCollectorInputDTO
from src.collectors.impl.muckrock.fetchers.foia.core import FOIAFetcher
from src.collectors.impl.muckrock.exceptions import MuckrockNoMoreDataError
from src.core.preprocessors.muckrock import MuckrockPreprocessor


Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
from src.collectors.enums import CollectorType
from src.collectors.source_collectors.base import AsyncCollectorBase
from src.collectors.source_collectors.muckrock.collectors.county.dto import MuckrockCountySearchCollectorInputDTO
from src.collectors.source_collectors.muckrock.fetch_requests.foia_loop import FOIALoopFetchRequest
from src.collectors.source_collectors.muckrock.fetch_requests.jurisdiction_loop import \
from src.collectors.impl.base import AsyncCollectorBase
from src.collectors.impl.muckrock.collectors.county.dto import MuckrockCountySearchCollectorInputDTO
from src.collectors.impl.muckrock.fetch_requests.foia_loop import FOIALoopFetchRequest
from src.collectors.impl.muckrock.fetch_requests.jurisdiction_loop import \
JurisdictionLoopFetchRequest
from src.collectors.source_collectors.muckrock.fetchers.foia.loop import FOIALoopFetcher
from src.collectors.source_collectors.muckrock.fetchers.jurisdiction.generator import \
from src.collectors.impl.muckrock.fetchers.foia.loop import FOIALoopFetcher
from src.collectors.impl.muckrock.fetchers.jurisdiction.generator import \
JurisdictionGeneratorFetcher
from src.core.preprocessors.muckrock import MuckrockPreprocessor

Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
import itertools

from src.collectors.enums import CollectorType
from src.collectors.source_collectors.base import AsyncCollectorBase
from src.collectors.source_collectors.muckrock.collectors.simple.dto import MuckrockSimpleSearchCollectorInputDTO
from src.collectors.source_collectors.muckrock.collectors.simple.searcher import FOIASearcher
from src.collectors.source_collectors.muckrock.fetchers.foia.core import FOIAFetcher
from src.collectors.source_collectors.muckrock.exceptions import SearchCompleteException
from src.collectors.impl.base import AsyncCollectorBase
from src.collectors.impl.muckrock.collectors.simple.dto import MuckrockSimpleSearchCollectorInputDTO
from src.collectors.impl.muckrock.collectors.simple.searcher import FOIASearcher
from src.collectors.impl.muckrock.fetchers.foia.core import FOIAFetcher
from src.collectors.impl.muckrock.exceptions import SearchCompleteException
from src.core.preprocessors.muckrock import MuckrockPreprocessor


Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from typing import Optional

from src.collectors.source_collectors.muckrock.fetchers.foia.core import FOIAFetcher
from src.collectors.source_collectors.muckrock.exceptions import SearchCompleteException
from src.collectors.impl.muckrock.fetchers.foia.core import FOIAFetcher
from src.collectors.impl.muckrock.exceptions import SearchCompleteException


class FOIASearcher:
Expand Down
6 changes: 6 additions & 0 deletions src/collectors/impl/muckrock/fetch_requests/foia.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
from src.collectors.impl.muckrock.fetch_requests.base import FetchRequest

Check warning on line 1 in src/collectors/impl/muckrock/fetch_requests/foia.py

View workflow job for this annotation

GitHub Actions / flake8

[flake8] src/collectors/impl/muckrock/fetch_requests/foia.py#L1 <100>

Missing docstring in public module
Raw output
./src/collectors/impl/muckrock/fetch_requests/foia.py:1:1: D100 Missing docstring in public module


class FOIAFetchRequest(FetchRequest):

Check warning on line 4 in src/collectors/impl/muckrock/fetch_requests/foia.py

View workflow job for this annotation

GitHub Actions / flake8

[flake8] src/collectors/impl/muckrock/fetch_requests/foia.py#L4 <101>

Missing docstring in public class
Raw output
./src/collectors/impl/muckrock/fetch_requests/foia.py:4:1: D101 Missing docstring in public class
page: int
page_size: int
5 changes: 5 additions & 0 deletions src/collectors/impl/muckrock/fetch_requests/foia_loop.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
from src.collectors.impl.muckrock.fetch_requests.base import FetchRequest

Check warning on line 1 in src/collectors/impl/muckrock/fetch_requests/foia_loop.py

View workflow job for this annotation

GitHub Actions / flake8

[flake8] src/collectors/impl/muckrock/fetch_requests/foia_loop.py#L1 <100>

Missing docstring in public module
Raw output
./src/collectors/impl/muckrock/fetch_requests/foia_loop.py:1:1: D100 Missing docstring in public module


class FOIALoopFetchRequest(FetchRequest):

Check warning on line 4 in src/collectors/impl/muckrock/fetch_requests/foia_loop.py

View workflow job for this annotation

GitHub Actions / flake8

[flake8] src/collectors/impl/muckrock/fetch_requests/foia_loop.py#L4 <101>

Missing docstring in public class
Raw output
./src/collectors/impl/muckrock/fetch_requests/foia_loop.py:4:1: D101 Missing docstring in public class
jurisdiction: int
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
from src.collectors.impl.muckrock.fetch_requests.base import FetchRequest

Check warning on line 1 in src/collectors/impl/muckrock/fetch_requests/jurisdiction_by_id.py

View workflow job for this annotation

GitHub Actions / flake8

[flake8] src/collectors/impl/muckrock/fetch_requests/jurisdiction_by_id.py#L1 <100>

Missing docstring in public module
Raw output
./src/collectors/impl/muckrock/fetch_requests/jurisdiction_by_id.py:1:1: D100 Missing docstring in public module


class JurisdictionByIDFetchRequest(FetchRequest):

Check warning on line 4 in src/collectors/impl/muckrock/fetch_requests/jurisdiction_by_id.py

View workflow job for this annotation

GitHub Actions / flake8

[flake8] src/collectors/impl/muckrock/fetch_requests/jurisdiction_by_id.py#L4 <101>

Missing docstring in public class
Raw output
./src/collectors/impl/muckrock/fetch_requests/jurisdiction_by_id.py:4:1: D101 Missing docstring in public class
jurisdiction_id: int
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from src.collectors.source_collectors.muckrock.fetch_requests.base import FetchRequest
from src.collectors.impl.muckrock.fetch_requests.base import FetchRequest

Check warning on line 1 in src/collectors/impl/muckrock/fetch_requests/jurisdiction_loop.py

View workflow job for this annotation

GitHub Actions / flake8

[flake8] src/collectors/impl/muckrock/fetch_requests/jurisdiction_loop.py#L1 <100>

Missing docstring in public module
Raw output
./src/collectors/impl/muckrock/fetch_requests/jurisdiction_loop.py:1:1: D100 Missing docstring in public module


class JurisdictionLoopFetchRequest(FetchRequest):
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from src.collectors.source_collectors.muckrock.fetch_requests.foia import FOIAFetchRequest
from src.collectors.source_collectors.muckrock.fetchers.templates.fetcher import MuckrockFetcherBase
from src.collectors.source_collectors.muckrock.constants import BASE_MUCKROCK_URL
from src.collectors.impl.muckrock.fetch_requests.foia import FOIAFetchRequest

Check warning on line 1 in src/collectors/impl/muckrock/fetchers/foia/core.py

View workflow job for this annotation

GitHub Actions / flake8

[flake8] src/collectors/impl/muckrock/fetchers/foia/core.py#L1 <100>

Missing docstring in public module
Raw output
./src/collectors/impl/muckrock/fetchers/foia/core.py:1:1: D100 Missing docstring in public module
from src.collectors.impl.muckrock.fetchers.templates.fetcher import MuckrockFetcherBase
from src.collectors.impl.muckrock.constants import BASE_MUCKROCK_URL

FOIA_BASE_URL = f"{BASE_MUCKROCK_URL}/foia"

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from src.collectors.source_collectors.muckrock.fetch_requests import FOIALoopFetchRequest
from src.collectors.source_collectors.muckrock.fetchers.foia.manager import FOIAFetchManager
from src.collectors.source_collectors.muckrock.fetchers.templates.generator import MuckrockGeneratorFetcher
from src.collectors.impl.muckrock.fetch_requests import FOIALoopFetchRequest

Check warning on line 1 in src/collectors/impl/muckrock/fetchers/foia/generator.py

View workflow job for this annotation

GitHub Actions / flake8

[flake8] src/collectors/impl/muckrock/fetchers/foia/generator.py#L1 <100>

Missing docstring in public module
Raw output
./src/collectors/impl/muckrock/fetchers/foia/generator.py:1:1: D100 Missing docstring in public module
from src.collectors.impl.muckrock.fetchers.foia.manager import FOIAFetchManager
from src.collectors.impl.muckrock.fetchers.templates.generator import MuckrockGeneratorFetcher


class FOIAGeneratorFetcher(MuckrockGeneratorFetcher):
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
from datasets import tqdm

from src.collectors.source_collectors.muckrock.fetch_requests.foia_loop import FOIALoopFetchRequest
from src.collectors.source_collectors.muckrock.fetchers.foia.manager import FOIAFetchManager
from src.collectors.source_collectors.muckrock.fetchers.templates.loop import MuckrockLoopFetcher
from src.collectors.impl.muckrock.fetch_requests.foia_loop import FOIALoopFetchRequest
from src.collectors.impl.muckrock.fetchers.foia.manager import FOIAFetchManager
from src.collectors.impl.muckrock.fetchers.templates.loop import MuckrockLoopFetcher


class FOIALoopFetcher(MuckrockLoopFetcher):
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from src.collectors.source_collectors.muckrock.fetch_requests.foia_loop import FOIALoopFetchRequest
from src.collectors.source_collectors.muckrock.constants import BASE_MUCKROCK_URL
from src.collectors.impl.muckrock.fetch_requests.foia_loop import FOIALoopFetchRequest

Check warning on line 1 in src/collectors/impl/muckrock/fetchers/foia/manager.py

View workflow job for this annotation

GitHub Actions / flake8

[flake8] src/collectors/impl/muckrock/fetchers/foia/manager.py#L1 <100>

Missing docstring in public module
Raw output
./src/collectors/impl/muckrock/fetchers/foia/manager.py:1:1: D100 Missing docstring in public module
from src.collectors.impl.muckrock.constants import BASE_MUCKROCK_URL


class FOIAFetchManager:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from src.collectors.source_collectors.muckrock.fetch_requests.jurisdiction_by_id import \
from src.collectors.impl.muckrock.fetch_requests.jurisdiction_by_id import \

Check warning on line 1 in src/collectors/impl/muckrock/fetchers/jurisdiction/core.py

View workflow job for this annotation

GitHub Actions / flake8

[flake8] src/collectors/impl/muckrock/fetchers/jurisdiction/core.py#L1 <100>

Missing docstring in public module
Raw output
./src/collectors/impl/muckrock/fetchers/jurisdiction/core.py:1:1: D100 Missing docstring in public module
JurisdictionByIDFetchRequest
from src.collectors.source_collectors.muckrock.fetchers.templates.fetcher import MuckrockFetcherBase
from src.collectors.source_collectors.muckrock.constants import BASE_MUCKROCK_URL
from src.collectors.impl.muckrock.fetchers.templates.fetcher import MuckrockFetcherBase
from src.collectors.impl.muckrock.constants import BASE_MUCKROCK_URL


class JurisdictionByIDFetcher(MuckrockFetcherBase):
Expand Down
Loading