Update Spark version.

Author: yruslan

.github/workflows/python.yml

@@ -49,9 +49,9 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - name: setup poetry
-        uses: abatilo/actions-poetry@v2.1.6
+        uses: abatilo/actions-poetry@v3.0.2
         with:
-          poetry-version: 1.4.2
+          poetry-version: 2.1.3
       - name: install dependencies
         run: make --silent install
       - name: test

pramen-py/.env.ci

@@ -34,5 +34,5 @@ export PRAMENPY_SPARK_CONFIG='spark.master=local,
                               spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog,
                               spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension,
                               spark.sql.session.timeZone=Africa/Johannesburg,
-                              spark.jars.packages=io.delta:delta-core_2.12:1.0.1,
+                              spark.jars.packages=io.delta:delta-spark_2.12:3.3.1,
                               spark.jars.repositories=https://maven-central.storage-download.googleapis.com/maven2/'

pramen-py/.env.local

@@ -34,6 +34,6 @@ export PRAMENPY_SPARK_CONFIG='spark.master=local,
                               spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog,
                               spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension,
                               spark.sql.session.timeZone=Africa/Johannesburg,
-                              spark.jars.packages=io.delta:delta-core_2.12:1.0.1,
+                              spark.jars.packages=io.delta:delta-spark_2.12:3.3.1,
                               spark.jars.repositories=https://maven-central.storage-download.googleapis.com/maven2/,
                               spark.sql.shuffle.partitions=1'

pramen-py/poetry.lock

@@ -117,7 +117,7 @@ dependencies = [
     "PyYAML>=6.0",
     "cattrs>=1.0.0; python_version<'3.7'",
     "cattrs>=22.1.0; python_version>='3.7'",
-    "pyspark==3.1.3",
+    "pyspark==3.5.7",
     "loguru>=0.6.0",
     "pytest==6.2.5",
     "pytest-asyncio==0.16",