Adiciona suporte aos parâmetros verify, issn e stop ao longo do pipeline de coleta de artigos, issues, journals

article/controller.py

@@ -445,6 +445,9 @@ def __init__(
         limit=None,
         timeout=None,
         opac_url=None,
+        verify=True,
+        issn=None,
+        stop=None,
         force_update=None,
     ):
         self.user = user
@@ -460,6 +463,9 @@ def __init__(
         self.limit = limit
         self.timeout = timeout
         self.opac_url = opac_url
+        self.verify = verify
+        self.issn = issn
+        self.stop = stop
         self.force_update = force_update
 
         self._iter_from_harvest_count = 0
@@ -492,18 +498,23 @@ def _iter_from_pid_provider(self):
             issn_list = [i for i in journal_issns if i] if journal_issns else None
             if journal_issns and not issn_list:
                 continue
-            qs = PidProviderXML.get_queryset(
+
+            kwargs = dict(
                 issn_list=issn_list,
                 from_pub_year=self.from_pub_year,
                 until_pub_year=self.until_pub_year,
                 from_updated_date=self.from_date,
                 until_updated_date=self.until_date,
                 proc_status_list=self.proc_status_list or [PPXML_STATUS_TODO, PPXML_STATUS_INVALID],
             )
-            self._iter_from_pid_provider_count += qs.count()
+            qs = PidProviderXML.get_queryset(**kwargs)
+            total = qs.count()
+
+            logging.info(f"PidProviderXML queryset total: {total}, kwargs: {kwargs}")
+
+            self._iter_from_pid_provider_count += total
             for item in qs.iterator():
                 yield {"pp_xml_id": item.id}
-        logging.info(f"_iter_from_pid_provider: yielded {self._iter_from_pid_provider_count} items")
 
     def _iter_from_article(self):
         """
@@ -550,7 +561,7 @@ def _iter_from_harvest(self):
         """Itera documentos coletados via OPAC ou ArticleMeta."""
 
         if Collection.objects.count() == 0:
-            Collection.load(self.user)
+            Collection.load(self.user, verify=self.verify)
 
         count = 0
         for collection_acron in self.collection_acron_list or list(Collection.get_acronyms()):
@@ -594,6 +605,9 @@ def _build_harvester(self, collection_acron):
             until_date=self.until_date,
             limit=self.limit,
             timeout=self.timeout,
+            verify=self.verify,
+            issn=self.issn,
+            stop=self.stop,
         )
         if collection_acron == "scl":
             return OPACHarvester(self.opac_url or "www.scielo.br", collection_acron, **kwargs)

article/sources/preprint.py

@@ -19,10 +19,16 @@
 class PreprintArticleSaveError(Exception): ...
 
 
-def harvest_preprints(URL, user):
+def harvest_preprints(URL, user, verify=True, timeout=30, stop=None):
     sickle = Sickle(URL)
     recs = sickle.ListRecords(metadataPrefix="oai_dc")
+
+    count = 0
     for rec in recs:
+        if stop and count >= stop:
+            logging.info(f"Reached stop limit of {stop} preprints")
+            break
+
         article_info = get_info_article(rec)
         identifier = get_doi(article_info["identifier"])
         doi = get_or_create_doi(doi=identifier, user=user)
@@ -73,6 +79,8 @@ def harvest_preprints(URL, user):
             # acessíveis na área administrativa
             # para que o usuário fique sabendo quais itens falharam
             raise PreprintArticleSaveError(e)
+
+        count += 1
 
 
 def get_info_article(rec):

article/sources/xmlsps.py

@@ -232,7 +232,7 @@ def load_article(user, xml=None, file_path=None, v3=None, pp_xml=None):
             article.languages.add(main_lang)
 
         article.sections.set(
-            get_or_create_toc_sections(xmltree=xmltree, user=user, errors=errors, issue=article.issue)
+            get_or_create_toc_sections(xmltree=xmltree, user=user, errors=errors, issue=article.issue, verify=True)
         )
         article.titles.set(
             create_or_update_titles(
@@ -522,7 +522,7 @@ def get_or_create_fundings(xmltree, user, item, errors):
     return data
 
 
-def get_or_create_toc_sections(xmltree, user, errors, issue):
+def get_or_create_toc_sections(xmltree, user, errors, issue, verify=True):
     """
     Extrai e cria seções do sumário (TOC) a partir do XML.
 
@@ -538,7 +538,7 @@ def get_or_create_toc_sections(xmltree, user, errors, issue):
     try:
         if not issue.table_of_contents.exists():
             for am_issue in AMIssue.objects.filter(new_record=issue):
-                load_issue_sections(user, issue, am_issue=am_issue)
+                load_issue_sections(user, issue, am_issue=am_issue, verify=verify)
         toc_sections = ArticleTocSections(xmltree=xmltree).sections
         for item in toc_sections:
             section_title = item.get("section")

article/tasks.py

@@ -51,7 +51,7 @@ def load_funding_data(user, file_path):
 
 
 @celery_app.task(bind=True, name=_('load_preprints'))
-def load_preprint(self, user_id, oai_pmh_preprint_uri):
+def load_preprint(self, user_id, oai_pmh_preprint_uri, verify=True, timeout=30, stop=None):
     """
     Coleta e carrega preprints de um endpoint OAI-PMH específico.
 
@@ -62,6 +62,9 @@ def load_preprint(self, user_id, oai_pmh_preprint_uri):
         self: Instância da tarefa Celery
         user_id (int): ID do usuário executando a tarefa (obrigatório)
         oai_pmh_preprint_uri (str): URI do endpoint OAI-PMH para coleta (obrigatório)
+        verify (bool): Verificação SSL para requisições HTTP
+        timeout (int): Timeout para requisições HTTP
+        stop (int, optional): Número máximo de preprints a processar
 
     Returns:
         None
@@ -89,7 +92,7 @@ def load_preprint(self, user_id, oai_pmh_preprint_uri):
     """
     user = User.objects.get(pk=user_id)
     ## fazer filtro para não coletar tudo sempre
-    harvest_preprints(oai_pmh_preprint_uri, user)
+    harvest_preprints(oai_pmh_preprint_uri, user, verify=verify, timeout=timeout, stop=stop)
 
 
 @celery_app.task(bind=True)
@@ -643,6 +646,7 @@ def task_check_article_availability(
     article_id=None,
     collection_acron_list=None,
     timeout=None,
+    verify=True,
     is_activate=None,
     force_update=False,
 ):
@@ -659,6 +663,7 @@ def task_check_article_availability(
         article_id (int, optional): ID do artigo a verificar (obrigatório)
         collection_acron_list (list, optional): Lista de acrônimos de coleções para filtro
         timeout (int, optional): Timeout em segundos para verificações HTTP
+        verify (bool): Verificação SSL para requisições HTTP
         is_activate (bool, optional): Se deve ativar artigo após verificação
         force_update (bool): Força nova verificação mesmo se recente
 
@@ -719,6 +724,9 @@ def task_dispatch_articles(
     limit=None,
     timeout=None,
     opac_url=None,
+    verify=True,
+    issn=None,
+    stop=None,
     # --- ativa article_source ---
     article_source_status_list=None,
 ):
@@ -747,6 +755,9 @@ def task_dispatch_articles(
         limit (int, optional): Limite máximo de artigos a processar
         timeout (int, optional): Timeout para operações HTTP
         opac_url (str, optional): URL base do OPAC para harvest
+        verify (bool): Verificação SSL para requisições HTTP
+        issn (str, optional): ISSN para filtrar por journal específico
+        stop (int, optional): Número máximo de itens a processar
         article_source_status_list (list, optional): Status do article_source para filtro
 
     Returns:
@@ -795,12 +806,14 @@ def task_dispatch_articles(
             limit=limit,
             timeout=timeout,
             opac_url=opac_url,
+            verify=verify,
+            issn=issn,
+            stop=stop,
             force_update=force_update,
         ):
             if item_kwargs is None:
                 skipped += 1
                 continue
-            logging.info(f"Dispatching article with kwargs: {item_kwargs}")
             task_process_article_pipeline.delay(**item_kwargs, **common_kwargs)
             dispatched += 1
 
@@ -937,7 +950,7 @@ def task_process_article_pipeline(
                 am_article=am_article,
                 auto_solve_pid_conflict=auto_solve_pid_conflict,
             )
-            pp_xml_id = article_source.pid_provider_xml.id
+            pp_xml_id = article_source.pid_provider_xml.id if article_source.pid_provider_xml else None
 
         if article_source_id:
             article_source = ArticleSource.objects.get(id=article_source_id)
@@ -946,7 +959,7 @@ def task_process_article_pipeline(
                 force_update=force_update,
                 auto_solve_pid_conflict=auto_solve_pid_conflict,
             )
-            pp_xml_id = article_source.pid_provider_xml.id
+            pp_xml_id = article_source.pid_provider_xml.id if article_source.pid_provider_xml else None
 
         if not pp_xml_id:
             raise ValueError(

bigbang/tasks.py

@@ -38,11 +38,12 @@ def task_start(
     self,
     user_id=None,
     username=None,
+    verify=True,
 ):
     try:
         user = _get_user(user_id, username)
         Language.load(user)
-        Collection.load(user)
+        Collection.load(user, verify=verify)
         Vocabulary.load(user)
         Standard.load(user)
         Subject.load(user)

bigbang/tasks_scheduler.py

@@ -141,7 +141,10 @@ def schedule_task_dispatch_articles(username, enabled=False):
             limit=None,
             timeout=None,
             opac_url=None,
+            verify=True,
+            issn=None,
             article_source_status_list=None,
+            stop=None,
         ),
         description=_("Dispatch articles to processing pipeline"),
         priority=TASK_PRIORITY,
@@ -271,7 +274,7 @@ def schedule_load_journal_from_article_meta(username, enabled=False):
     """
     Agenda a tarefa de carga de dados de journals obtidos do AM e Core.
 
-    Configura verify=True para verificação SSL nas requisições HTTP.
+    Configura verify=True para verificação SSL e limit=100 para limitar coleta.
     """
     schedule_task(
         task="journal.tasks.load_journal_from_article_meta",
@@ -280,6 +283,9 @@ def schedule_load_journal_from_article_meta(username, enabled=False):
             load_data=False,
             collection_acron="scl",
             verify=True,
+            limit=1000,
+            issn=None,
+            stop=None,
         ),
         description=_("Carga de dados de journals obtidos do AM e Core"),
         priority=1,
@@ -295,7 +301,7 @@ def schedule_collect_journals_from_am(username, enabled=False):
     """
     Agenda a tarefa de coleta de journals da fonte AM.
 
-    Configura verify=True para verificação SSL nas requisições HTTP.
+    Configura verify=True para verificação SSL e limit=100 para limitar coleta.
     """
     schedule_task(
         task="journal.tasks.load_journal_from_article_meta",
@@ -304,6 +310,9 @@ def schedule_collect_journals_from_am(username, enabled=False):
             load_data=True,
             collection_acron="scl",
             verify=True,
+            limit=1000,
+            issn=None,
+            stop=None,
         ),
         description=_("Coleta de journals da fonte AM"),
         priority=1,
@@ -416,7 +425,9 @@ def schedule_export_journal_to_articlemeta(username, enabled=False):
 
 def schedule_load_issue_from_articlemeta(username, enabled=False):
     """
-    Agenda a tarefa de carregar issues do ArticleMeta
+    Agenda a tarefa de carregar issues do ArticleMeta.
+
+    Configura verify=True para verificação SSL e limit=100 para limitar coleta.
     """
     schedule_task(
         task="issue.tasks.load_issue_from_articlemeta",
@@ -429,6 +440,10 @@ def schedule_load_issue_from_articlemeta(username, enabled=False):
             until_date=None,
             force_update=None,
             timeout=30,
+            verify=True,
+            limit=1000,
+            issn=None,
+            stop=None,
         ),
         description=_("Load issues from ArticleMeta"),
         priority=TASK_PRIORITY,

collection/models.py

@@ -223,12 +223,12 @@ def __str__(self):
     base_form_class = CoreAdminModelForm
 
     @classmethod
-    def load(cls, user, collections_data=None):
+    def load(cls, user, collections_data=None, verify=True):
         if not collections_data:
             collections_data = fetch_data(
                 "https://articlemeta.scielo.org/api/v1/collection/identifiers/",
                 json=True,
-                verify=False,
+                verify=verify,
             )
 
         for collection_data in collections_data:

collection/tasks.py

@@ -8,9 +8,9 @@
 
 
 @celery_app.task(bind=True)
-def task_load_collections(self, user_id=None, username=None):
+def task_load_collections(self, user_id=None, username=None, verify=True):
     if user_id:
         user = User.objects.get(pk=user_id)
     if username:
         user = User.objects.get(username=username)
-    Collection.load(user)
+    Collection.load(user, verify=verify)