Upgrade duckdb and raise maximum tested python version (#65)

brycekbargar · web-flow · commit 156e165ee891 · 2026-02-26T11:02:07.000-05:00
The last PR caught that the changes for the server side transform aren't compatible with older versions of duckdb. It was already getting unwieldy and so I chose to make the necessary changes in another PR. This raises the minimum supported version of DuckDB and also starts testing against python 3.14. There has been an ongoing cache contention issue resulting in long build times that is also resolved here.
diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
@@ -21,8 +21,8 @@ jobs:
       - uses: pdm-project/setup-pdm@v4
         with:
           cache: true
-          cache-dependency-path: pylock.toml
-      - run: pdm install -G:all --lockfile pylock.toml
+          cache-dependency-path: pylock.lint.toml
+      - run: pdm install -G:all --lockfile pylock.lint.toml
       - uses: jaxxstorm/action-install-gh-release@v1.10.0
         with:
           repo: houseabsolute/precious
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -51,7 +51,7 @@ jobs:
       # install dependencies
       - uses: pdm-project/setup-pdm@v4
         with:
-          python-version: 3.13
+          python-version: 3.14
           cache: true
           cache-dependency-path: pylock.maximal.toml
       - run: pdm install -G:all --lockfile pylock.maximal.toml
diff --git a/environment.yaml b/environment.yaml
@@ -5,6 +5,6 @@ channels:
   - nodefaults
 dependencies:
   - python>=3.10,<3.11
-  - pdm==2.25.5
-  - precious==0.10.0
+  - pdm==2.26.6
+  - precious==0.10.2
   - libpq>=13.0
diff --git a/pylock.lint.toml b/pylock.lint.toml
diff --git a/pylock.maximal.toml b/pylock.maximal.toml
diff --git a/pylock.minimal.toml b/pylock.minimal.toml
diff --git a/pylock.toml b/pylock.toml
diff --git a/pyproject.toml b/pyproject.toml
@@ -11,9 +11,8 @@ authors = [
   { name = "Nassib Nassar", email = "nassib@indexdata.com" },
 ]
 dependencies = [
-  # Versions lower than 0.9 have issues binding the custom functions
-  # 1.4 has a number of regressions so I'm letting it shake out until some fixes
-  "duckdb>=0.9.2,<1.4",
+  # 1.3 introduces json each
+  "duckdb>=1.3",
   # 4.64 is the first release with type stubs
   "tqdm>=4.64.0",
   # 0.3 allows checking the path for id based paging
@@ -67,6 +66,9 @@ include = ["src/**"]
 distribution = true
 package-dir = "src"
 
+[tool.pdm.resolution]
+allow-prereleases = false
+
 [tool.pdm.scripts]
 test.composite = [
   "rm -f .coverage",
@@ -76,7 +78,8 @@ test.composite = [
 lock.composite = [
   "rm -f pylock.toml pylock.maximal.toml pylock.minimal.toml",
   "pdm lock --python=3.10",
-  "pdm lock --lockfile pylock.maximal.toml --python=3.13",
+  "cp pylock.toml pylock.lint.toml",
+  "pdm lock --lockfile pylock.maximal.toml --python=3.14",
   "pdm lock --strategy direct_minimal_versions --lockfile pylock.minimal.toml --python=3.10",
 ]
 test-install.composite = [
diff --git a/src/ldlite/_csv.py b/src/ldlite/_csv.py
@@ -64,6 +64,7 @@ def to_csv(
                         s += ","
                     if attrs[i][1] in [
                         "NUMBER",
+                        "DECIMAL(18,3)",
                         "bigint",
                         "numeric",
                         20,
diff --git a/src/ldlite/database/_duckdb.py b/src/ldlite/database/_duckdb.py
@@ -24,13 +24,6 @@ def __init__(self, db: duckdb.DuckDBPyConnection) -> None:
 
     @staticmethod
     def _setup_jfuncs(conn: duckdb.DuckDBPyConnection) -> None:
-        with conn.cursor() as cur:
-            cur.execute("SELECT string_split(ltrim(version(),'v'), '.') AS has_lambda;")
-            if ver := cur.fetchone():
-                (ma, mi, _) = ver[0]
-                if int(ma) > 1 and int(mi) >= 3:
-                    cur.execute("SET lambda_syntax = 'ENABLE_SINGLE_ARROW';")
-
         with conn.cursor() as cur:
             cur.execute(
                 r"""
@@ -63,8 +56,8 @@ def _setup_jfuncs(conn: duckdb.DuckDBPyConnection) -> None:
             END
         WHEN 'array' THEN
             CASE
-                WHEN length(list_filter((main.json_extract(j, p))::JSON[], x -> x != 'null'::JSON)) = 0 THEN 'null'::JSON
-                ELSE list_filter((main.json_extract(j, p))::JSON[], x -> x != 'null'::JSON)
+                WHEN length(list_filter((main.json_extract(j, p))::JSON[], lambda x : x != 'null'::JSON)) = 0 THEN 'null'::JSON
+                ELSE list_filter((main.json_extract(j, p))::JSON[], lambda x : x != 'null'::JSON)
             END
         ELSE coalesce(main.json_extract(j, p), 'null'::JSON)
     END
diff --git a/tests/test_expansion.py b/tests/test_expansion.py
@@ -218,17 +218,32 @@ def case_basic_array() -> ExpansionTC:
             Assertion("""SELECT COUNT(*) FROM tests.prefix__t__list2""", expect=2),
             Assertion(
                 """
-SELECT id, list1_o, list1
+SELECT id, list1
+FROM tests.prefix__t__list1
+ORDER BY id, list1
+                """,
+                expect=[
+                    ("id1", "a1"),
+                    ("id1", "b1"),
+                    ("id1", "c1"),
+                    ("id2", "a2"),
+                    ("id2", "b2"),
+                    ("id2", "c2"),
+                ],
+            ),
+            Assertion(
+                """
+SELECT id, list1_o
 FROM tests.prefix__t__list1
 ORDER BY id, list1_o
                 """,
                 expect=[
-                    ("id1", 1, "a1"),
-                    ("id1", 2, "b1"),
-                    ("id1", 3, "c1"),
-                    ("id2", 1, "a2"),
-                    ("id2", 2, "b2"),
-                    ("id2", 3, "c2"),
+                    ("id1", 1),
+                    ("id1", 2),
+                    ("id1", 3),
+                    ("id2", 1),
+                    ("id2", 2),
+                    ("id2", 3),
                 ],
             ),
             *[
@@ -281,15 +296,28 @@ def case_nested_arrays() -> ExpansionTC:
             Assertion("""SELECT COUNT(*) FROM tests.prefix__t__sub""", expect=4),
             Assertion(
                 """
-SELECT id, sub_o, sub__id
+SELECT id, sub__id
+FROM tests.prefix__t__sub
+ORDER BY id, sub__id
+                """,
+                expect=[
+                    ("id1", "sub1id1"),
+                    ("id1", "sub2id1"),
+                    ("id2", "sub1id2"),
+                    ("id2", "sub2id2"),
+                ],
+            ),
+            Assertion(
+                """
+SELECT id, sub_o
 FROM tests.prefix__t__sub
 ORDER BY id, sub_o
                 """,
                 expect=[
-                    ("id1", 1, "sub1id1"),
-                    ("id1", 2, "sub2id1"),
-                    ("id2", 1, "sub1id2"),
-                    ("id2", 2, "sub2id2"),
+                    ("id1", 1),
+                    ("id1", 2),
+                    ("id2", 1),
+                    ("id2", 2),
                 ],
             ),
             Assertion(
diff --git a/tests/test_json_operators.py b/tests/test_json_operators.py
@@ -7,7 +7,7 @@
 import duckdb
 import psycopg
 import pytest
-from pytest_cases import parametrize, parametrize_with_cases
+from pytest_cases import get_case_id, parametrize, parametrize_with_cases
 
 if TYPE_CHECKING:
     from _typeshed import dbapi
@@ -38,9 +38,6 @@ class JsonTC:
         ("obj_some", '{"k1":"v1","k2":null}'),
         ("obj_empty", "null"),
         ("arr_zero", "null"),
-        ("arr_str", '["s1","s2","s3"]'),
-        ("arr_str_some", '["s1","s2"]'),
-        ("arr_obj_some", '[{"k1":"v1"}]'),
         ("na", "null"),
         ("na_str1", "null"),
         ("na_str2", "null"),
@@ -55,6 +52,43 @@ def case_jextract(p: tuple[Any, ...]) -> JsonTC:
     )
 
 
+# Duckdb through 1.3 and 1.4 have different json comparison behavior here
+# Whitespace matters in 1.4 and not 1.3
+# This makes the arrays text and compares the values as a workaround
+@parametrize(
+    p=[
+        ("arr_str", '["s1", "s2", "s3"]'),
+        ("arr_str_some", '["s1", "s2"]'),
+        ("arr_obj_some", '[{"k1":"v1"}]'),
+    ],
+)
+def case_jextract_duckdb(p: tuple[Any, ...]) -> JsonTC:
+    return JsonTC(
+        """SELECT ldlite_system.jextract(jc, $1){assertion} FROM j;""",
+        p[:1],
+        """::text[] = $2::JSON::text[]""",
+        p[1:],
+    )
+
+
+# The differences betweeen postgres/duckdb here only matters for tests
+# This can all be rectified when duckdb 1.4 is the minimum version
+@parametrize(
+    p=[
+        ("arr_str", '["s1", "s2", "s3"]'),
+        ("arr_str_some", '["s1", "s2"]'),
+        ("arr_obj_some", '[{"k1":"v1"}]'),
+    ],
+)
+def case_jextract_postgres(p: tuple[Any, ...]) -> JsonTC:
+    return JsonTC(
+        """SELECT ldlite_system.jextract(jc, $1){assertion} FROM j;""",
+        p[:1],
+        """ = $2::JSONB""",
+        p[1:],
+    )
+
+
 @parametrize(
     p=[
         ("str", "str_val"),
@@ -293,7 +327,11 @@ def duckdb_jop_dsn() -> Iterator[str]:
         yield dsn
 
 
-@parametrize_with_cases("tc", cases=".")
+@parametrize_with_cases(
+    "tc",
+    cases=".",
+    filter=lambda cf: "postgres" not in get_case_id(cf),
+)
 def test_duckdb(duckdb_jop_dsn: str, tc: JsonTC) -> None:
     from ldlite import LDLite
 
@@ -316,7 +354,11 @@ def pg_jop_dsn(pg_dsn: None | Callable[[str], str]) -> str:
     return dsn
 
 
-@parametrize_with_cases("tc", cases=".")
+@parametrize_with_cases(
+    "tc",
+    cases=".",
+    filter=lambda cf: "duckdb" not in get_case_id(cf),
+)
 def test_postgres(pg_jop_dsn: str, tc: JsonTC) -> None:
     from ldlite import LDLite
 
