Feature/data 2567 python upgrade dagger

.github/workflows/ci-data.yml

@@ -17,10 +17,10 @@ jobs:
         with:
           persist-credentials: false
 
-      - name: Set up Python 3.9
+      - name: Set up Python 3.12
         uses: actions/setup-python@v2
         with:
-          python-version: 3.9
+          python-version: 3.12
 
       - name: Install dependencies
         run: |

Makefile

@@ -96,17 +96,16 @@ install: clean ## install the package to the active Python's site-packages
 
 
 install-dev: clean ## install the package to the active Python's site-packages
-	virtualenv -p python3.9 venv; \
+	virtualenv -p python3.12 venv; \
 	source venv/bin/activate; \
-	python -m pip install --upgrade pip; \
-	python setup.py install; \
+	python -m pip install --upgrade pip setuptools wheel; \
 	pip install -e . ; \
 	SYSTEM_VERSION_COMPAT=0 CFLAGS='-std=c++20' pip install -r reqs/dev.txt -r reqs/test.txt
 
 install-test: clean ## install the package to the active Python's site-packages
-	virtualenv -p python3.9 venv; \
+	virtualenv -p python3.12 venv; \
 	source venv/bin/activate; \
-	python -m pip install --upgrade pip; \
+	python -m pip install --upgrade pip setuptools wheel; \
 	pip install -r reqs/test.txt -r reqs/base.txt
 
 install-ui: clean ## install the package to the active Python's site-packages

reqs/base.txt

@@ -1,4 +1,3 @@
-aiobotocore>=2.5.0
 click>=8.1.3
 croniter==2.0.2
 envyaml==1.10.211231

reqs/dev.txt

@@ -1,7 +1,6 @@
 pip==24.0
-apache-airflow[amazon,postgres,s3,statsd]==2.9.0 --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.0/constraints-3.9.txt"
+apache-airflow[amazon,postgres,s3,statsd]==2.11.0 --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.11.0/constraints-3.12.txt"
 black==22.10.0
-boto3==1.34.82
 bumpversion==0.6.0
 coverage==7.4.4
 elasticsearch==7.17.7

reqs/test.txt

@@ -1,4 +1,4 @@
-apache-airflow[amazon,postgres,s3,statsd]==2.9.0
+apache-airflow[amazon,postgres,s3,statsd]==2.11.0 --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.11.0/constraints-3.12.txt"
 pytest-cov==4.0.0
 pytest==7.2.0
 graphviz

reqs/ui.txt

@@ -4,3 +4,4 @@ flask==2.2.5
 python-dotenv==0.21.0
 requests==2.32.4
 WTForms==2.3.3
+

setup.py

@@ -45,7 +45,7 @@ def reqs(*f):
     classifiers=[
         "Development Status :: 2 - Pre-Alpha",
         "Intended Audience :: Developers",
-        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.12",
     ],
     description="Config Driven ETL",
     entry_points={"console_scripts": ["dagger=dagger.main:cli"]},

tests/fixtures/dag_creator/airflow/dag_test_external_sensor.dot

@@ -3,7 +3,7 @@ digraph test_external_sensor {
 	"dummy-control-flow" [color="#000000" fillcolor="#ffefeb" label="dummy-control-flow" shape=rectangle style="filled,rounded"]
 	dummy_first [color="#000000" fillcolor="#e8f7e4" label=dummy_first shape=rectangle style="filled,rounded"]
 	dummy_second [color="#000000" fillcolor="#e8f7e4" label=dummy_second shape=rectangle style="filled,rounded"]
-	"test_batch-batch-sensor" [color="#000000" fillcolor="#19647e" label="test_batch-batch-sensor" shape=rectangle style="filled,rounded"]
+	"test_batch-batch-sensor" [color="#000000" fillcolor="#4db7db" label="test_batch-batch-sensor" shape=rectangle style="filled,rounded"]
 	"dummy-control-flow" -> "test_batch-batch-sensor"
 	dummy_first -> dummy_second
 	"test_batch-batch-sensor" -> dummy_first