From 29203d309006bc9c89bb536d1e9cbbc6fa5854f0 Mon Sep 17 00:00:00 2001 From: David Siklosi Date: Mon, 3 Nov 2025 13:31:00 +0100 Subject: [PATCH 1/6] Upgrading local env --- Makefile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 8872dba..32f1f93 100644 --- a/Makefile +++ b/Makefile @@ -96,17 +96,16 @@ install: clean ## install the package to the active Python's site-packages install-dev: clean ## install the package to the active Python's site-packages - virtualenv -p python3.9 venv; \ + virtualenv -p python3.12 venv; \ source venv/bin/activate; \ - python -m pip install --upgrade pip; \ - python setup.py install; \ + python -m pip install --upgrade pip setuptools wheel; \ pip install -e . ; \ SYSTEM_VERSION_COMPAT=0 CFLAGS='-std=c++20' pip install -r reqs/dev.txt -r reqs/test.txt install-test: clean ## install the package to the active Python's site-packages - virtualenv -p python3.9 venv; \ + virtualenv -p python3.12 venv; \ source venv/bin/activate; \ - python -m pip install --upgrade pip; \ + python -m pip install --upgrade pip setuptools wheel; \ pip install -r reqs/test.txt -r reqs/base.txt install-ui: clean ## install the package to the active Python's site-packages From 2f665c5d3bbc7fcef099be09ba5a1a00b2c8b6dd Mon Sep 17 00:00:00 2001 From: David Siklosi Date: Mon, 3 Nov 2025 13:31:39 +0100 Subject: [PATCH 2/6] Fixing dependency issues --- reqs/base.txt | 1 - reqs/dev.txt | 3 +-- setup.py | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/reqs/base.txt b/reqs/base.txt index f88ab98..d5b400f 100644 --- a/reqs/base.txt +++ b/reqs/base.txt @@ -1,4 +1,3 @@ -aiobotocore>=2.5.0 click>=8.1.3 croniter==2.0.2 envyaml==1.10.211231 diff --git a/reqs/dev.txt b/reqs/dev.txt index 238b2e2..bfb4a44 100644 --- a/reqs/dev.txt +++ b/reqs/dev.txt @@ -1,7 +1,6 @@ pip==24.0 -apache-airflow[amazon,postgres,s3,statsd]==2.9.0 --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.0/constraints-3.9.txt" +apache-airflow[amazon,postgres,s3,statsd]==2.9.0 --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.0/constraints-3.12.txt" black==22.10.0 -boto3==1.34.82 bumpversion==0.6.0 coverage==7.4.4 elasticsearch==7.17.7 diff --git a/setup.py b/setup.py index f8b4b28..a9580a1 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ def reqs(*f): classifiers=[ "Development Status :: 2 - Pre-Alpha", "Intended Audience :: Developers", - "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.12", ], description="Config Driven ETL", entry_points={"console_scripts": ["dagger=dagger.main:cli"]}, From 57d300ab039ab944ee9f1186565cf56740424a52 Mon Sep 17 00:00:00 2001 From: David Siklosi Date: Mon, 3 Nov 2025 13:32:02 +0100 Subject: [PATCH 3/6] Upgrading git workflow python --- .github/workflows/ci-data.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-data.yml b/.github/workflows/ci-data.yml index bef5bed..2d3044c 100644 --- a/.github/workflows/ci-data.yml +++ b/.github/workflows/ci-data.yml @@ -17,10 +17,10 @@ jobs: with: persist-credentials: false - - name: Set up Python 3.9 + - name: Set up Python 3.12 uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.12 - name: Install dependencies run: | From 735279f40a861a14a430a4f75e5bad6fd9348fae Mon Sep 17 00:00:00 2001 From: David Siklosi Date: Mon, 3 Nov 2025 14:00:43 +0100 Subject: [PATCH 4/6] Bumping airflow version --- reqs/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reqs/dev.txt b/reqs/dev.txt index bfb4a44..b39d00a 100644 --- a/reqs/dev.txt +++ b/reqs/dev.txt @@ -1,5 +1,5 @@ pip==24.0 -apache-airflow[amazon,postgres,s3,statsd]==2.9.0 --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.0/constraints-3.12.txt" +apache-airflow[amazon,postgres,s3,statsd]==2.11.0 --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.11.0/constraints-3.12.txt" black==22.10.0 bumpversion==0.6.0 coverage==7.4.4 From d72b81230decd2ade25109b2560c79a8664cacf2 Mon Sep 17 00:00:00 2001 From: David Siklosi Date: Mon, 3 Nov 2025 14:15:07 +0100 Subject: [PATCH 5/6] Fixing version mismatch between test and dev --- reqs/test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reqs/test.txt b/reqs/test.txt index 7bdc89f..6bb6c2e 100644 --- a/reqs/test.txt +++ b/reqs/test.txt @@ -1,4 +1,4 @@ -apache-airflow[amazon,postgres,s3,statsd]==2.9.0 +apache-airflow[amazon,postgres,s3,statsd]==2.11.0 --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.11.0/constraints-3.12.txt" pytest-cov==4.0.0 pytest==7.2.0 graphviz From 08fb0ab8e88c85452d9e361645a32a2e644f5f2a Mon Sep 17 00:00:00 2001 From: David Siklosi Date: Mon, 3 Nov 2025 14:49:40 +0100 Subject: [PATCH 6/6] Fixing unit tests caused by airflow version bump --- reqs/ui.txt | 1 + tests/fixtures/dag_creator/airflow/dag_test_external_sensor.dot | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/reqs/ui.txt b/reqs/ui.txt index 10f95e3..46d5546 100644 --- a/reqs/ui.txt +++ b/reqs/ui.txt @@ -4,3 +4,4 @@ flask==2.2.5 python-dotenv==0.21.0 requests==2.32.4 WTForms==2.3.3 + diff --git a/tests/fixtures/dag_creator/airflow/dag_test_external_sensor.dot b/tests/fixtures/dag_creator/airflow/dag_test_external_sensor.dot index 9824739..1add9af 100644 --- a/tests/fixtures/dag_creator/airflow/dag_test_external_sensor.dot +++ b/tests/fixtures/dag_creator/airflow/dag_test_external_sensor.dot @@ -3,7 +3,7 @@ digraph test_external_sensor { "dummy-control-flow" [color="#000000" fillcolor="#ffefeb" label="dummy-control-flow" shape=rectangle style="filled,rounded"] dummy_first [color="#000000" fillcolor="#e8f7e4" label=dummy_first shape=rectangle style="filled,rounded"] dummy_second [color="#000000" fillcolor="#e8f7e4" label=dummy_second shape=rectangle style="filled,rounded"] - "test_batch-batch-sensor" [color="#000000" fillcolor="#19647e" label="test_batch-batch-sensor" shape=rectangle style="filled,rounded"] + "test_batch-batch-sensor" [color="#000000" fillcolor="#4db7db" label="test_batch-batch-sensor" shape=rectangle style="filled,rounded"] "dummy-control-flow" -> "test_batch-batch-sensor" dummy_first -> dummy_second "test_batch-batch-sensor" -> dummy_first