From 7c4fbf74785a8face55eb2cce5e5b566d1712c07 Mon Sep 17 00:00:00 2001 From: tadasbar Date: Thu, 13 Mar 2025 12:10:16 +0200 Subject: [PATCH 01/18] Add log retrieval after job finalization --- .../executor.py | 53 +++++++++++++++++-- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/snakemake_executor_plugin_googlebatch/executor.py b/snakemake_executor_plugin_googlebatch/executor.py index aab2ff3..ac4519e 100644 --- a/snakemake_executor_plugin_googlebatch/executor.py +++ b/snakemake_executor_plugin_googlebatch/executor.py @@ -1,4 +1,5 @@ import os +import time import uuid from typing import List @@ -11,9 +12,8 @@ import snakemake_executor_plugin_googlebatch.utils as utils import snakemake_executor_plugin_googlebatch.command as cmdutil -from google.api_core.exceptions import DeadlineExceeded -from google.cloud import batch_v1 - +from google.api_core.exceptions import DeadlineExceeded, ResourceExhausted +from google.cloud import batch_v1, logging class GoogleBatchExecutor(RemoteExecutor): def __post_init__(self): @@ -501,6 +501,9 @@ async def check_active_jobs(self, active_jobs: List[SubmittedJobInfo]): # SUCCEEDED # FAILED # DELETION_IN_PROGRESS + if response.status.state.name in ["FAILED", "SUCCEEDED"]: + self.save_finished_job_logs(j) + if response.status.state.name == "FAILED": msg = f"Google Batch job '{j.external_jobid}' failed. " self.report_job_error(j, msg=msg, aux_logs=aux_logs) @@ -512,9 +515,51 @@ async def check_active_jobs(self, active_jobs: List[SubmittedJobInfo]): else: yield j + def save_finished_job_logs( + self, + job_info: SubmittedJobInfo, + sleeps=60, + page_size=1000, + ): + """ + Download logs using Google Cloud Logging API and save + them locally. Since tail logging does not work, this function + is run only at the end of the job. + """ + job_uid = job_info.aux["batch_job"].uid + filter_query = f"labels.job_uid={job_uid}" + logfname = job_info.aux["logfile"] + + log_client = logging.Client(project=self.executor_settings.project) + logger = log_client.logger("batch_task_logs") + + try: + with open(logfname, "w", encoding="utf-8") as logfile: + for log_entry in logger.list_entries( + filter_=filter_query, + page_size=page_size, + ): + logfile.write(log_entry.payload + "\n") + except ResourceExhausted: + self.logger.warning("Too many requests to Google Logging API.\n" + + f"Skipping logs for job {job_uid} and sleeping for {sleeps}s.") + time.sleep(sleeps) + + self.logger.warning(f"Trying to retreive logs for job {job_uid} once more.") + try: + with open(logfname, "w", encoding="utf-8") as logfile: + for log_entry in logger.list_entries( + filter_=filter_query, + page_size=page_size, + ): + logfile.write(log_entry.payload + "\n") + except ResourceExhausted: + self.logger.warning("Retry to retrieve logs failed, " + + f"the log file {logfname} might be incomplete.") + def cancel_jobs(self, active_jobs: List[SubmittedJobInfo]): """ - cancel all active jobs. This method is called when snakemake is interrupted. + Cancel all active jobs. This method is called when snakemake is interrupted. """ for job in active_jobs: jobid = job.external_jobid From 83fc8257964af5ba3212e07c97d6434a294767a7 Mon Sep 17 00:00:00 2001 From: tadasbar Date: Thu, 13 Mar 2025 16:11:31 +0200 Subject: [PATCH 02/18] Reformat --- snakemake_executor_plugin_googlebatch/executor.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/snakemake_executor_plugin_googlebatch/executor.py b/snakemake_executor_plugin_googlebatch/executor.py index ac4519e..e0057cd 100644 --- a/snakemake_executor_plugin_googlebatch/executor.py +++ b/snakemake_executor_plugin_googlebatch/executor.py @@ -15,6 +15,7 @@ from google.api_core.exceptions import DeadlineExceeded, ResourceExhausted from google.cloud import batch_v1, logging + class GoogleBatchExecutor(RemoteExecutor): def __post_init__(self): # Attach variables for easy access @@ -541,8 +542,10 @@ def save_finished_job_logs( ): logfile.write(log_entry.payload + "\n") except ResourceExhausted: - self.logger.warning("Too many requests to Google Logging API.\n" + - f"Skipping logs for job {job_uid} and sleeping for {sleeps}s.") + self.logger.warning( + "Too many requests to Google Logging API.\n" + + f"Skipping logs for job {job_uid} and sleeping for {sleeps}s." + ) time.sleep(sleeps) self.logger.warning(f"Trying to retreive logs for job {job_uid} once more.") @@ -554,8 +557,10 @@ def save_finished_job_logs( ): logfile.write(log_entry.payload + "\n") except ResourceExhausted: - self.logger.warning("Retry to retrieve logs failed, " + - f"the log file {logfname} might be incomplete.") + self.logger.warning( + "Retry to retrieve logs failed, " + + f"the log file {logfname} might be incomplete." + ) def cancel_jobs(self, active_jobs: List[SubmittedJobInfo]): """ From dd69391c57587b64af5185c4c1410fc37e8d4204 Mon Sep 17 00:00:00 2001 From: tadasbar Date: Thu, 13 Mar 2025 16:22:47 +0200 Subject: [PATCH 03/18] Add myself to contributors --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f481bf5..173113d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,8 @@ version = "0.5.0" description = "" authors = [ "Vanessa Sochat ", - "Johannes Koester " + "Johannes Koester ", + "Tadas Bareikis ", ] readme = "README.md" license = "MIT" From 82011d9d316a58c726058cc631abc867bef81d55 Mon Sep 17 00:00:00 2001 From: tadasbar Date: Thu, 13 Mar 2025 16:25:30 +0200 Subject: [PATCH 04/18] Update tests --- .github/workflows/ci_mocked_api.yaml | 22 ++++++++++++++++------ tests/docker-compose.yml | 11 +++++++++++ 2 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 tests/docker-compose.yml diff --git a/.github/workflows/ci_mocked_api.yaml b/.github/workflows/ci_mocked_api.yaml index 0eea8eb..73f6cc9 100644 --- a/.github/workflows/ci_mocked_api.yaml +++ b/.github/workflows/ci_mocked_api.yaml @@ -83,13 +83,23 @@ jobs: - name: 'Use gcloud CLI' run: 'gcloud info' - # - id: 'auth' - # uses: 'google-github-actions/auth@v1' - # with: - # credentials_json: '${{ secrets.GCP_SA_KEY }}' + - name: Set up Docker Compose + run: | + sudo apt-get update + sudo apt-get install docker-compose + + - name: Start MinIO service + run: docker-compose -f tests/docker-compose.yml up -d - # - name: 'Set up GCloud SDK' - # uses: 'google-github-actions/setup-gcloud@v1' + - name: Install MinIO Client CLI + run: | + curl -O https://dl.min.io/client/mc/release/linux-amd64/mc + chmod +x mc + sudo mv mc /usr/local/bin/ + + - name: Configure MinIO client + run: | + mc alias set minio http://localhost:9000 minio minio123 - name: Run pytest run: | diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml new file mode 100644 index 0000000..e6207d1 --- /dev/null +++ b/tests/docker-compose.yml @@ -0,0 +1,11 @@ +version: '3.8' + +services: + minio: + image: minio/minio + ports: + - "9000:9000" + environment: + MINIO_ACCESS_KEY: minio + MINIO_SECRET_KEY: minio123 + command: server /data From 8e9b7c8d7f4f813421eabb7baf23653209241fd8 Mon Sep 17 00:00:00 2001 From: tadasbar Date: Thu, 13 Mar 2025 17:24:34 +0200 Subject: [PATCH 05/18] Fix pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 173113d..1def308 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ description = "" authors = [ "Vanessa Sochat ", "Johannes Koester ", - "Tadas Bareikis ", + "Tadas Bareikis " ] readme = "README.md" license = "MIT" From b1b5bc18960d009bc3d4ad97ee5c7bd7944a4508 Mon Sep 17 00:00:00 2001 From: tadasbar Date: Thu, 13 Mar 2025 17:32:58 +0200 Subject: [PATCH 06/18] Update pyproject --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1def308..e7080a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,15 +22,15 @@ google-cloud-storage = "^2.12.0" snakemake-interface-common = "^1.14.0" snakemake-interface-executor-plugins = "^9.0.0" jinja2 = "^3.1.2" -google-cloud-logging = "^3.8.0" +google-cloud-logging = "^3.11.4" [tool.poetry.group.dev.dependencies] black = "^24.4.0" flake8 = "^6.1.0" coverage = "^7.3.1" pytest = "^7.4.2" -snakemake = "^8.18.0" -snakemake-storage-plugin-s3 = "^0.2.10" +snakemake = "^8.30.0" +snakemake-storage-plugin-s3 = "^0.3.1" [tool.coverage.run] omit = [".*", "*/site-packages/*", "Snakefile"] From 13dd0b1f77c7fa851d70a182313946ae50222322 Mon Sep 17 00:00:00 2001 From: tadasbar Date: Thu, 13 Mar 2025 18:00:36 +0200 Subject: [PATCH 07/18] Overcome a test case --- snakemake_executor_plugin_googlebatch/executor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/snakemake_executor_plugin_googlebatch/executor.py b/snakemake_executor_plugin_googlebatch/executor.py index e0057cd..3f1012b 100644 --- a/snakemake_executor_plugin_googlebatch/executor.py +++ b/snakemake_executor_plugin_googlebatch/executor.py @@ -531,6 +531,10 @@ def save_finished_job_logs( filter_query = f"labels.job_uid={job_uid}" logfname = job_info.aux["logfile"] + # Handle the test case, when no job_uid is returned + if job_uid == "": + return + log_client = logging.Client(project=self.executor_settings.project) logger = log_client.logger("batch_task_logs") From 8da7caa6bbdb964e3362297dc91db2f5587ed786 Mon Sep 17 00:00:00 2001 From: tadasbar Date: Thu, 13 Mar 2025 18:26:26 +0200 Subject: [PATCH 08/18] Update test --- snakemake_executor_plugin_googlebatch/executor.py | 4 ---- tests/tests_mocked_api.py | 9 ++++++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/snakemake_executor_plugin_googlebatch/executor.py b/snakemake_executor_plugin_googlebatch/executor.py index 3f1012b..e0057cd 100644 --- a/snakemake_executor_plugin_googlebatch/executor.py +++ b/snakemake_executor_plugin_googlebatch/executor.py @@ -531,10 +531,6 @@ def save_finished_job_logs( filter_query = f"labels.job_uid={job_uid}" logfname = job_info.aux["logfile"] - # Handle the test case, when no job_uid is returned - if job_uid == "": - return - log_client = logging.Client(project=self.executor_settings.project) logger = log_client.logger("batch_task_logs") diff --git a/tests/tests_mocked_api.py b/tests/tests_mocked_api.py index fe8d7fd..ebbdb33 100644 --- a/tests/tests_mocked_api.py +++ b/tests/tests_mocked_api.py @@ -11,7 +11,7 @@ class TestWorkflowsMockedApi(TestWorkflowsBase): @patch( "google.cloud.batch_v1.BatchServiceClient.create_job", new=MagicMock( - return_value=Job(name="foo"), + return_value=Job(name="foo", uid="bar"), autospec=True, ), ) @@ -22,6 +22,13 @@ class TestWorkflowsMockedApi(TestWorkflowsBase): autospec=True, ), ) + @patch( + "google.cloud.logging.Client.logger", + new=MagicMock( + return_value=MagicMock(list_entries=lambda filter_, page_size: []), + autospec=True, + ), + ) @patch( "snakemake.dag.DAG.check_and_touch_output", new=AsyncMock(autospec=True), From dcab2c6953957ad3ebd4575f0fd22a1d830fd893 Mon Sep 17 00:00:00 2001 From: tadasbar Date: Thu, 13 Mar 2025 20:00:21 +0200 Subject: [PATCH 09/18] Update test - remove docker-compose yaml --- .github/workflows/ci_mocked_api.yaml | 8 ++++++-- tests/docker-compose.yml | 11 ----------- 2 files changed, 6 insertions(+), 13 deletions(-) delete mode 100644 tests/docker-compose.yml diff --git a/.github/workflows/ci_mocked_api.yaml b/.github/workflows/ci_mocked_api.yaml index 73f6cc9..ea863f8 100644 --- a/.github/workflows/ci_mocked_api.yaml +++ b/.github/workflows/ci_mocked_api.yaml @@ -88,8 +88,12 @@ jobs: sudo apt-get update sudo apt-get install docker-compose - - name: Start MinIO service - run: docker-compose -f tests/docker-compose.yml up -d + - name: Setup minio + uses: comfuture/minio-action@v1 + with: + access_key: minio + secret_key: minio123 + port: 9000 - name: Install MinIO Client CLI run: | diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml deleted file mode 100644 index e6207d1..0000000 --- a/tests/docker-compose.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: '3.8' - -services: - minio: - image: minio/minio - ports: - - "9000:9000" - environment: - MINIO_ACCESS_KEY: minio - MINIO_SECRET_KEY: minio123 - command: server /data From a1a835aea9aa27975f08468dc3af910e2238bc7b Mon Sep 17 00:00:00 2001 From: tadasbar Date: Thu, 13 Mar 2025 20:16:40 +0200 Subject: [PATCH 10/18] Remove the docker-compose install --- .github/workflows/ci_mocked_api.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ci_mocked_api.yaml b/.github/workflows/ci_mocked_api.yaml index ea863f8..31436fd 100644 --- a/.github/workflows/ci_mocked_api.yaml +++ b/.github/workflows/ci_mocked_api.yaml @@ -83,11 +83,6 @@ jobs: - name: 'Use gcloud CLI' run: 'gcloud info' - - name: Set up Docker Compose - run: | - sudo apt-get update - sudo apt-get install docker-compose - - name: Setup minio uses: comfuture/minio-action@v1 with: From ede0946c7dfb2ac721534bb22f93469d68e8aaca Mon Sep 17 00:00:00 2001 From: tadasbar Date: Thu, 13 Mar 2025 20:17:20 +0200 Subject: [PATCH 11/18] Minor spelling fix --- snakemake_executor_plugin_googlebatch/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snakemake_executor_plugin_googlebatch/executor.py b/snakemake_executor_plugin_googlebatch/executor.py index e0057cd..bb180fa 100644 --- a/snakemake_executor_plugin_googlebatch/executor.py +++ b/snakemake_executor_plugin_googlebatch/executor.py @@ -548,7 +548,7 @@ def save_finished_job_logs( ) time.sleep(sleeps) - self.logger.warning(f"Trying to retreive logs for job {job_uid} once more.") + self.logger.warning(f"Trying to retrieve logs for job {job_uid} once more.") try: with open(logfname, "w", encoding="utf-8") as logfile: for log_entry in logger.list_entries( From 511d6d1f6980f9ce6dcd716a0b725cd0ecf9b5e6 Mon Sep 17 00:00:00 2001 From: tadasbar Date: Thu, 13 Mar 2025 21:17:38 +0200 Subject: [PATCH 12/18] Wrap payload into str --- snakemake_executor_plugin_googlebatch/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snakemake_executor_plugin_googlebatch/executor.py b/snakemake_executor_plugin_googlebatch/executor.py index bb180fa..cc86431 100644 --- a/snakemake_executor_plugin_googlebatch/executor.py +++ b/snakemake_executor_plugin_googlebatch/executor.py @@ -540,7 +540,7 @@ def save_finished_job_logs( filter_=filter_query, page_size=page_size, ): - logfile.write(log_entry.payload + "\n") + logfile.write(str(log_entry.payload) + "\n") except ResourceExhausted: self.logger.warning( "Too many requests to Google Logging API.\n" From 45c2d278f4ab4c340584283b4764efbf75691bd5 Mon Sep 17 00:00:00 2001 From: tadasbar Date: Thu, 13 Mar 2025 21:31:01 +0200 Subject: [PATCH 13/18] Add missing str() on payload --- snakemake_executor_plugin_googlebatch/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snakemake_executor_plugin_googlebatch/executor.py b/snakemake_executor_plugin_googlebatch/executor.py index cc86431..2f834d0 100644 --- a/snakemake_executor_plugin_googlebatch/executor.py +++ b/snakemake_executor_plugin_googlebatch/executor.py @@ -555,7 +555,7 @@ def save_finished_job_logs( filter_=filter_query, page_size=page_size, ): - logfile.write(log_entry.payload + "\n") + logfile.write(str(log_entry.payload) + "\n") except ResourceExhausted: self.logger.warning( "Retry to retrieve logs failed, " From a6f776d519fcd6a86a9332e2ed80604c6ecfd0f1 Mon Sep 17 00:00:00 2001 From: tadasbar Date: Fri, 14 Mar 2025 10:53:11 +0200 Subject: [PATCH 14/18] Remove unnecessary CI steps --- .github/workflows/ci_mocked_api.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/ci_mocked_api.yaml b/.github/workflows/ci_mocked_api.yaml index 31436fd..6193948 100644 --- a/.github/workflows/ci_mocked_api.yaml +++ b/.github/workflows/ci_mocked_api.yaml @@ -90,16 +90,6 @@ jobs: secret_key: minio123 port: 9000 - - name: Install MinIO Client CLI - run: | - curl -O https://dl.min.io/client/mc/release/linux-amd64/mc - chmod +x mc - sudo mv mc /usr/local/bin/ - - - name: Configure MinIO client - run: | - mc alias set minio http://localhost:9000 minio minio123 - - name: Run pytest run: | poetry run coverage run -m pytest tests/tests_mocked_api.py -v From 1fabd6df879395a776b38ed4793ff53ad27f783c Mon Sep 17 00:00:00 2001 From: tadasbar Date: Sat, 5 Apr 2025 19:36:14 +0300 Subject: [PATCH 15/18] Encapsulate logging attempt in a function; print a msg when saving logs. --- .../executor.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/snakemake_executor_plugin_googlebatch/executor.py b/snakemake_executor_plugin_googlebatch/executor.py index d2d6c0f..1d68d5c 100644 --- a/snakemake_executor_plugin_googlebatch/executor.py +++ b/snakemake_executor_plugin_googlebatch/executor.py @@ -541,13 +541,18 @@ def save_finished_job_logs( log_client = logging.Client(project=self.executor_settings.project) logger = log_client.logger("batch_task_logs") - try: - with open(logfname, "w", encoding="utf-8") as logfile: + def attempt_log_save(fname, query, page_size): + with open(fname, "w", encoding="utf-8") as logfile: for log_entry in logger.list_entries( - filter_=filter_query, + filter_=query, page_size=page_size, ): logfile.write(str(log_entry.payload) + "\n") + + self.logger.info(f"Saving logs for Batch job {job_uid} to {logfname}.") + + try: + attempt_log_save(logfname, filter_query, page_size) except ResourceExhausted: self.logger.warning( "Too many requests to Google Logging API.\n" @@ -557,12 +562,7 @@ def save_finished_job_logs( self.logger.warning(f"Trying to retrieve logs for job {job_uid} once more.") try: - with open(logfname, "w", encoding="utf-8") as logfile: - for log_entry in logger.list_entries( - filter_=filter_query, - page_size=page_size, - ): - logfile.write(str(log_entry.payload) + "\n") + attempt_log_save(logfname, filter_query, page_size) except ResourceExhausted: self.logger.warning( "Retry to retrieve logs failed, " From 7e643d69f61b144ec74f8834a76e6ff37470554b Mon Sep 17 00:00:00 2001 From: tadasbar Date: Sat, 5 Apr 2025 19:59:56 +0300 Subject: [PATCH 16/18] Add "Batch" specifier to error messages --- snakemake_executor_plugin_googlebatch/executor.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/snakemake_executor_plugin_googlebatch/executor.py b/snakemake_executor_plugin_googlebatch/executor.py index 1d68d5c..fbb291a 100644 --- a/snakemake_executor_plugin_googlebatch/executor.py +++ b/snakemake_executor_plugin_googlebatch/executor.py @@ -560,7 +560,9 @@ def attempt_log_save(fname, query, page_size): ) time.sleep(sleeps) - self.logger.warning(f"Trying to retrieve logs for job {job_uid} once more.") + self.logger.warning( + f"Trying to retrieve logs for Batch job {job_uid} once more." + ) try: attempt_log_save(logfname, filter_query, page_size) except ResourceExhausted: @@ -568,6 +570,10 @@ def attempt_log_save(fname, query, page_size): "Retry to retrieve logs failed, " + f"the log file {logfname} might be incomplete." ) + except Exception as e: + self.logger.warning( + f"Failed to retrieve logs for Batch job {job_uid}: {str(e)}" + ) def cancel_jobs(self, active_jobs: List[SubmittedJobInfo]): """ From 65d5b264aa6d516443dfe9b53fc0b03b01458246 Mon Sep 17 00:00:00 2001 From: tadasbar Date: Sat, 5 Apr 2025 20:12:21 +0300 Subject: [PATCH 17/18] Move the logger into the argument list --- snakemake_executor_plugin_googlebatch/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snakemake_executor_plugin_googlebatch/executor.py b/snakemake_executor_plugin_googlebatch/executor.py index fbb291a..17e4bdb 100644 --- a/snakemake_executor_plugin_googlebatch/executor.py +++ b/snakemake_executor_plugin_googlebatch/executor.py @@ -541,7 +541,7 @@ def save_finished_job_logs( log_client = logging.Client(project=self.executor_settings.project) logger = log_client.logger("batch_task_logs") - def attempt_log_save(fname, query, page_size): + def attempt_log_save(fname, logger, query, page_size): with open(fname, "w", encoding="utf-8") as logfile: for log_entry in logger.list_entries( filter_=query, From bc49e38587761591b35a27bd322d03ace8edd271 Mon Sep 17 00:00:00 2001 From: tadasbar Date: Sat, 5 Apr 2025 20:24:14 +0300 Subject: [PATCH 18/18] Add the missing logger argument. --- snakemake_executor_plugin_googlebatch/executor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/snakemake_executor_plugin_googlebatch/executor.py b/snakemake_executor_plugin_googlebatch/executor.py index 17e4bdb..13b5a4a 100644 --- a/snakemake_executor_plugin_googlebatch/executor.py +++ b/snakemake_executor_plugin_googlebatch/executor.py @@ -552,7 +552,7 @@ def attempt_log_save(fname, logger, query, page_size): self.logger.info(f"Saving logs for Batch job {job_uid} to {logfname}.") try: - attempt_log_save(logfname, filter_query, page_size) + attempt_log_save(logfname, logger, filter_query, page_size) except ResourceExhausted: self.logger.warning( "Too many requests to Google Logging API.\n" @@ -564,7 +564,7 @@ def attempt_log_save(fname, logger, query, page_size): f"Trying to retrieve logs for Batch job {job_uid} once more." ) try: - attempt_log_save(logfname, filter_query, page_size) + attempt_log_save(logfname, logger, filter_query, page_size) except ResourceExhausted: self.logger.warning( "Retry to retrieve logs failed, "