Remove restriction on the number of subtasks (#19)

* Update dependencies

* Update dataclasses to latest protobuf messages

* Merge future tasks to submissions

* Switch to merged task submissions message

* Add additional type checks

* Update Changelog

* Re-record runner tests grpc calls

* Filter out python 3.10 google.api_core future warnings

Author: lukasbindreiter

.pre-commit-config.yaml

@@ -4,15 +4,17 @@ repos:
     hooks:
       - id: check-yaml
       - id: end-of-file-fixer
+  - repo: https://github.com/tsvikas/sync-with-uv
+    rev: v0.4.0
+    hooks:
+      - id: sync-with-uv
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    # keep the version here in sync with the version in uv.lock
-    rev: "v0.13.0"
+    rev: v0.14.3
     hooks:
       - id: ruff-check
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
   - repo: https://github.com/RobertCraigie/pyright-python
-    # keep the version here in sync with the version in uv.lock
     rev: v1.1.400
     hooks:
       - id: pyright

CHANGELOG.md

@@ -7,6 +7,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.45.0] - 2025-11-17
+
+### Added
+
+- `tilebox-workflows`: Added `execution_stats` to the `Job` object to provide programmatic access to a job's execution
+  statistics.
+- `tilebox-workflows`: Added query filters to the `JobClient.query` method to filter jobs by multiple automation ids,
+  job state, and job name.
+- `tilebox-workflows`: Added additional JobState values to indicate a job's current state and progress more accurately.
+- `tilebox-workflows`: Removed the restriction of `64` subtasks per task.
+
+### Changed
+
+- `tilebox-workflows`: Deprecated the `job.canceled` property, which will be removed in a future version. The
+  equivalent expression behavior for the old `job.canceled` is `(job.state in (JobState.CANCELED, JobState.FAILED)`.
+- `tilebox-workflows`: Deprecated the `job.started_at` property, which will be removed in a future version. The
+  equivalent information is available in `job.execution_stats.first_task_started_at` instead.
+- `tilebox-workflows`: The `JobState.QUEUED` enum value has been renamed in favor of the more accurate
+  `JobState.SUBMITTED`. The `QUEUED` value is still available as an alias for `SUBMITTED` for backwards compatibility,
+  but will be removed in a future version.
+- `tilebox-workflows`: Switched to an updated internal `TaskSubmission` message format that allows for more efficient
+  submission of a very large number of tasks.
+
 ## [0.44.0] - 2025-09-18
 
 ### Added
@@ -266,7 +289,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Released under the [MIT](https://opensource.org/license/mit) license.
 - Released packages: `tilebox-datasets`, `tilebox-workflows`, `tilebox-storage`, `tilebox-grpc`
 
-[Unreleased]: https://github.com/tilebox/tilebox-python/compare/v0.44.0...HEAD
+[Unreleased]: https://github.com/tilebox/tilebox-python/compare/v0.45.0...HEAD
+[0.45.0]: https://github.com/tilebox/tilebox-python/compare/v0.45.0...v0.45.0
 [0.44.0]: https://github.com/tilebox/tilebox-python/compare/v0.43.0...v0.44.0
 [0.43.0]: https://github.com/tilebox/tilebox-python/compare/v0.42.0...v0.43.0
 [0.42.0]: https://github.com/tilebox/tilebox-python/compare/v0.41.0...v0.42.0

tests.sh

@@ -13,7 +13,15 @@ for package in $packages; do
     if [ -d _tilebox ]; then
         module=_tilebox
     fi
-    uv run --all-packages pytest -Wall -Werror --cov=$module --cov-branch -v --junitxml=test-report.xml . || exit 1
+
+    PYTHON_VERSION=$(uv run python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null || true)
+
+    if [ "$PYTHON_VERSION" = "3.10" ]; then
+        # ignore: FutureWarning: You are using a Python version (3.10.11) which Google will stop supporting in new releases of google.api_core once it reaches its end of life (2026-10-04).
+        uv run --all-packages pytest -Wall -Werror -W "ignore::FutureWarning" --cov=$module --cov-branch -v --junitxml=test-report.xml . || exit 1
+    else
+        uv run --all-packages pytest -Wall -Werror --cov=$module --cov-branch -v --junitxml=test-report.xml . || exit 1
+    fi
 
     cd .. || exit 1 # cd back to the root of the monorepo
 done

tilebox-datasets/tests/test_timeseries.py

@@ -1,10 +1,10 @@
+from dataclasses import dataclass
 from datetime import datetime, timedelta
 from unittest.mock import MagicMock, patch
 from uuid import uuid4
 
 import pytest
 import xarray as xr
-from attr import dataclass
 from hypothesis import assume, given, settings
 from hypothesis.stateful import Bundle, RuleBasedStateMachine, consumes, invariant, rule
 from hypothesis.strategies import lists

tilebox-datasets/tilebox/datasets/datasets/v1/core_pb2.py

@@ -112,7 +112,7 @@ class CollectionInfos(_message.Message):
     def __init__(self, data: _Optional[_Iterable[_Union[CollectionInfo, _Mapping]]] = ...) -> None: ...
 
 class Dataset(_message.Message):
-    __slots__ = ("id", "group_id", "type", "code_name", "name", "summary", "icon", "description", "permissions", "visibility", "slug", "type_editable")
+    __slots__ = ("id", "group_id", "type", "code_name", "name", "summary", "icon", "description", "permissions", "visibility", "slug", "type_editable", "collections")
     ID_FIELD_NUMBER: _ClassVar[int]
     GROUP_ID_FIELD_NUMBER: _ClassVar[int]
     TYPE_FIELD_NUMBER: _ClassVar[int]
@@ -125,6 +125,7 @@ class Dataset(_message.Message):
     VISIBILITY_FIELD_NUMBER: _ClassVar[int]
     SLUG_FIELD_NUMBER: _ClassVar[int]
     TYPE_EDITABLE_FIELD_NUMBER: _ClassVar[int]
+    COLLECTIONS_FIELD_NUMBER: _ClassVar[int]
     id: _id_pb2.ID
     group_id: _id_pb2.ID
     type: _dataset_type_pb2.AnnotatedType
@@ -137,7 +138,8 @@ class Dataset(_message.Message):
     visibility: Visibility
     slug: str
     type_editable: bool
-    def __init__(self, id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., group_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., type: _Optional[_Union[_dataset_type_pb2.AnnotatedType, _Mapping]] = ..., code_name: _Optional[str] = ..., name: _Optional[str] = ..., summary: _Optional[str] = ..., icon: _Optional[str] = ..., description: _Optional[str] = ..., permissions: _Optional[_Iterable[_Union[DatasetPermission, str]]] = ..., visibility: _Optional[_Union[Visibility, str]] = ..., slug: _Optional[str] = ..., type_editable: bool = ...) -> None: ...
+    collections: _containers.RepeatedCompositeFieldContainer[CollectionInfo]
+    def __init__(self, id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., group_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., type: _Optional[_Union[_dataset_type_pb2.AnnotatedType, _Mapping]] = ..., code_name: _Optional[str] = ..., name: _Optional[str] = ..., summary: _Optional[str] = ..., icon: _Optional[str] = ..., description: _Optional[str] = ..., permissions: _Optional[_Iterable[_Union[DatasetPermission, str]]] = ..., visibility: _Optional[_Union[Visibility, str]] = ..., slug: _Optional[str] = ..., type_editable: bool = ..., collections: _Optional[_Iterable[_Union[CollectionInfo, _Mapping]]] = ...) -> None: ...
 
 class DatasetGroup(_message.Message):
     __slots__ = ("id", "parent_id", "code_name", "name", "icon")

tilebox-datasets/tilebox/datasets/datasets/v1/core_pb2.pyi

@@ -7,12 +7,12 @@
 
 from _tilebox.grpc.error import NotFoundError
 from tilebox.datasets.query.time_interval import datetime_to_timestamp
-from tilebox.workflows.data import Job, uuid_message_to_uuid, uuid_to_uuid_message
+from tilebox.workflows.data import Job, JobState, uuid_message_to_uuid, uuid_to_uuid_message
 from tilebox.workflows.jobs.client import JobClient
 from tilebox.workflows.jobs.service import JobService
 from tilebox.workflows.task import ExecutionContext, Task
 from tilebox.workflows.workflows.v1.core_pb2 import Job as JobMessage
-from tilebox.workflows.workflows.v1.core_pb2 import JobState
+from tilebox.workflows.workflows.v1.core_pb2 import JobState as JobStateEnum
 from tilebox.workflows.workflows.v1.diagram_pb2 import Diagram
 from tilebox.workflows.workflows.v1.job_pb2 import (
     CancelJobRequest,
@@ -45,8 +45,7 @@ def SubmitJob(self, req: SubmitJobRequest) -> JobMessage:  # noqa: N802
             id=uuid_to_uuid_message(job_id),
             name=req.job_name,
             trace_parent=req.trace_parent,
-            canceled=False,
-            state=JobState.JOB_STATE_QUEUED,
+            state=JobStateEnum.JOB_STATE_SUBMITTED,
             submitted_at=datetime_to_timestamp(datetime.now(tz=timezone.utc)),
         )
         self.jobs[job_id] = job
@@ -65,8 +64,7 @@ def RetryJob(self, req: RetryJobRequest) -> RetryJobResponse:  # noqa: N802
             raise NotFoundError(f"No such job: {job_id}")
 
         copy = JobMessage.FromString(self.jobs[job_id].SerializeToString())
-        copy.canceled = False
-        copy.state = JobState.JOB_STATE_QUEUED
+        copy.state = JobStateEnum.JOB_STATE_SUBMITTED
         self.jobs[job_id] = copy
         return RetryJobResponse(num_tasks_rescheduled=1)
 
@@ -76,8 +74,7 @@ def CancelJob(self, req: CancelJobRequest) -> CancelJobResponse:  # noqa: N802
             raise NotFoundError(f"No such job: {job_id}")
 
         copy = JobMessage.FromString(self.jobs[job_id].SerializeToString())
-        copy.canceled = True
-        copy.state = JobState.JOB_STATE_STARTED
+        copy.state = JobStateEnum.JOB_STATE_CANCELED
         self.jobs[job_id] = copy
         return CancelJobResponse()
 
@@ -87,7 +84,7 @@ def VisualizeJob(self, req: VisualizeJobRequest) -> Diagram:  # noqa: N802
             raise NotFoundError(f"No such job: {job_id}")
 
         job = self.jobs[job_id]
-        if job.canceled:
+        if job.state == JobStateEnum.JOB_STATE_CANCELED:
             return Diagram(svg=b"<svg><text>Job canceled</text></svg>")
 
         return Diagram(svg=b"<svg><text>Job queued</text></svg>")
@@ -129,26 +126,26 @@ def submit_job(self, job: Job) -> Job:
     def get_queued_job(self, job: Job) -> None:
         got = self.job_client.find(job.id)
         assert got.name == job.name
-        assert not got.canceled
+        assert got.state == JobState.SUBMITTED
 
     @rule(job=cancelled_jobs)
     def get_canceled_job(self, job: Job) -> None:
         got = self.job_client.find(job.id)
         assert got.name == job.name
-        assert got.canceled
+        assert got.state == JobState.CANCELED
 
     @rule(target=cancelled_jobs, job=consumes(queued_jobs))  # consumes -> remove from bundle afterwards
     def cancel_job(self, job: Job) -> Job:
         self.job_client.cancel(job.id)
         job = self.job_client.find(job.id)
-        assert job.canceled
+        assert job.state == JobState.CANCELED
         return job
 
     @rule(target=queued_jobs, job=consumes(cancelled_jobs))  # consumes -> remove from bundle afterwards
     def retry_job(self, job: Job) -> Job:
         self.job_client.retry(job.id)
         job = self.job_client.find(job.id)
-        assert not job.canceled
+        assert job.state == JobState.SUBMITTED
         return job
 
     @rule()