Tests cleanup, sanitized runs on CI

.github/workflows/SQLiteTests.yml

@@ -59,6 +59,86 @@ jobs:
     - name: Test extension
       env: 
         LOCAL_EXTENSION_REPO: ${{ github.workspace }}/build/release/repository/
+        SQLITE_TPCH_GENERATED: 1
       run: |
+        make data/db/tpch.db
         ./build/release/test/unittest 'test/*'
 
+  linux-sanitized:
+    name: Linux ${{ matrix.sanitizer.name }} (amd64)
+    needs: linux
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        sanitizer:
+        - name: ASan
+          threadsan_enabled: 0
+        - name: TSan
+          threadsan_enabled: 1
+
+    env:
+      CMAKE_BUILD_PARALLEL_LEVEL: 2
+      GEN: ninja
+      CC: 'ccache gcc'
+      CXX: 'ccache g++'
+      CCACHE_DIR: ${{ github.workspace }}/ccache
+
+    steps:
+
+    - name: Free disk space
+      uses: endersonmenezes/free-disk-space@v2.1.1
+      continue-on-error: true
+      with:
+        remove_haskell: true
+        remove_tool_cache: true
+        remove_folders: "/usr/local/share/powershell /usr/share/swift"
+        testing: false
+
+    - name: Checkout
+      uses: actions/checkout@v6
+      with:
+        submodules: 'true'
+
+    - name: Install required ubuntu packages
+      run: |
+        sudo apt-get update -y -q -o=Dpkg::Use-Pty=0
+        sudo apt-get install -y -q -o=Dpkg::Use-Pty=0 \
+          build-essential \
+          ccache \
+          cmake \
+          mold \
+          ninja-build
+
+    - name: Cache Key
+      id: cache_key
+      working-directory: ./duckdb
+      run: |
+        DUCKDB_VERSION=$(git rev-parse --short HEAD)
+        KEY="${{ runner.os }}-${{ runner.arch }}-$DUCKDB_VERSION"-relassert-${{ matrix.sanitizer.name }}
+        echo "value=${KEY}" >> "${GITHUB_OUTPUT}"
+
+    - name: Restore Cache
+      uses: actions/cache/restore@v5
+      with:
+        path: ${{ github.workspace }}/ccache
+        key: ${{ steps.cache_key.outputs.value }}
+
+    - name: Build extension
+      env:
+        THREADSAN: ${{ matrix.sanitizer.threadsan_enabled }}
+      run: |
+        make relassert
+        cat ./build/relassert/build.ninja | grep fsanitize | head -n 1
+
+    - name: Save Cache
+      uses: actions/cache/save@v5
+      with:
+        path: ${{ github.workspace }}/ccache
+        key: ${{ steps.cache_key.outputs.value }}
+
+    - name: Test extension
+      env: 
+        LOCAL_EXTENSION_REPO: ${{ github.workspace }}/build/relassert/repository/
+      run: |
+        ./build/relassert/test/unittest

extension_config.cmake

@@ -3,7 +3,6 @@
 # Extension from this repo
 duckdb_extension_load(sqlite_scanner
     SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}
-    LOAD_TESTS
 )
 
 # Any extra extensions that should be built

scripts/time_tests.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import subprocess
+import time
+import glob
+
+def run_and_time_unit_tests():
+    """
+    Discovers and runs unit tests, records their elapsed time, and returns
+    a list of test results sorted by elapsed time in descending order.
+    """
+    # Assuming the script is run from the project root (e.g., duckdb-postgres/)
+    project_root = os.getcwd()
+
+    # Define paths to the unittest executable and extension repository
+    # These paths are relative to the project_root
+    unittest_executable_path = os.path.join(project_root, 'build', 'release', 'test', 'unittest')
+    extension_repo_path = os.path.join(project_root, 'build', 'release', 'repository')
+    test_directory = os.path.join(project_root, 'test')
+
+    # --- Pre-flight checks ---
+    if not os.path.exists(unittest_executable_path):
+        print(f"Error: Unittest executable not found at '{unittest_executable_path}'. "
+              "Please ensure the project is built and the path is correct.")
+        return []
+
+    if not os.path.exists(extension_repo_path):
+        print(f"Warning: Extension repository not found at '{extension_repo_path}'. "
+              "Tests requiring extensions might fail. Please ensure it exists if needed.")
+
+    if not os.path.isdir(test_directory):
+        print(f"Error: Test directory not found at '{test_directory}'. "
+              "Please ensure the 'test/' directory exists in the project root.")
+        return []
+
+    # Discover all .test files recursively within the 'test/' directory
+    test_files = glob.glob(os.path.join(test_directory, '**', '*.test'), recursive=True)
+
+    if not test_files:
+        print(f"No '.test' files found in '{test_directory}'.")
+        return []
+
+    print(f"Found {len(test_files)} test files. Running them one by one...\n")
+
+    results = []
+    # Prepare the environment variables for the subprocess
+    # Copy current environment and add/override LOCAL_EXTENSION_REPO
+    env_vars = os.environ.copy()
+    env_vars['LOCAL_EXTENSION_REPO'] = extension_repo_path
+
+    test_files = sorted(test_files)
+
+    for test_file_full_path in test_files:
+        # Get the path relative to the project root, as required by the unittest executable
+        test_name = os.path.relpath(test_file_full_path, project_root)
+        command = [unittest_executable_path, test_name]
+
+        print(f"--- Running test: {test_name} ---")
+        start_time = time.monotonic()
+        process_returncode = -1 # Default to error state
+
+        try:
+            # Execute the command. capture_output=True captures stdout/stderr.
+            # text=True decodes stdout/stderr as text.
+            # check=False prevents an exception for non-zero exit codes, allowing us to record time for failed tests.
+            process = subprocess.run(
+                command,
+                env=env_vars,
+                capture_output=True,
+                text=True,
+                check=False
+            )
+            end_time = time.monotonic()
+            elapsed_time = end_time - start_time
+            process_returncode = process.returncode
+
+            results.append({
+                'test_name': test_name,
+                'elapsed_time': elapsed_time,
+                'returncode': process_returncode
+            })
+
+            if process_returncode != 0:
+                print(f"  Test FAILED (Return Code: {process_returncode}) - Elapsed: {elapsed_time:.4f} seconds")
+                # Uncomment the following lines to see stdout/stderr for failed tests
+                # print("  --- STDOUT ---")
+                # print(process.stdout.strip())
+                # print("  --- STDERR ---")
+                # print(process.stderr.strip())
+            else:
+                print(f"  Test PASSED - Elapsed: {elapsed_time:.4f} seconds")
+
+        except FileNotFoundError:
+            print(f"  Error: Unittest executable not found or command failed to start for '{test_name}'.")
+            results.append({'test_name': test_name, 'elapsed_time': -1.0, 'returncode': -1})
+        except Exception as e:
+            print(f"  An unexpected error occurred while running '{test_name}': {e}")
+            results.append({'test_name': test_name, 'elapsed_time': -1.0, 'returncode': -1})
+        print("-" * (len(test_name) + 18)) # Separator for clarity
+
+    # Sort results by elapsed time in descending order
+    # Tests that errored out (elapsed_time = -1.0) will appear at the end due to sorting behavior
+    sorted_results = sorted(results, key=lambda x: x['elapsed_time'], reverse=True)
+
+    print("\n\n--- Test Summary (Sorted by Elapsed Time, Descending) ---")
+    if not sorted_results:
+        print("No test results to display.")
+    else:
+        for result in sorted_results:
+            status = "PASSED" if result['returncode'] == 0 else "FAILED" if result['returncode'] != -1 else "ERROR"
+            if result['elapsed_time'] >= 0:
+                print(f"{result['test_name']}: {result['elapsed_time']:.4f} seconds ({status})")
+            else:
+                print(f"{result['test_name']}: Error during execution ({status})")
+
+    return sorted_results
+
+if __name__ == "__main__":
+    run_and_time_unit_tests()

test/sql/scanner/tpch.test

@@ -14,28 +14,28 @@ require tpch
 loop i 1 9
 
 query I
-PRAGMA tpch(${i})
+PRAGMA tpch({i})
 ----
-<FILE>:duckdb/extension/tpch/dbgen/answers/sf0.1/q0${i}.csv
+<FILE>:duckdb/extension/tpch/dbgen/answers/sf0.1/q0{i}.csv
 
 endloop
 
 
 loop i 10 15
 
 query I
-PRAGMA tpch(${i})
+PRAGMA tpch({i})
 ----
-<FILE>:duckdb/extension/tpch/dbgen/answers/sf0.1/q${i}.csv
+<FILE>:duckdb/extension/tpch/dbgen/answers/sf0.1/q{i}.csv
 
 endloop
 
 loop i 16 23
 
 query I
-PRAGMA tpch(${i})
+PRAGMA tpch({i})
 ----
-<FILE>:duckdb/extension/tpch/dbgen/answers/sf0.1/q${i}.csv
+<FILE>:duckdb/extension/tpch/dbgen/answers/sf0.1/q{i}.csv
 
 endloop
 

test/sql/storage/attach_alter.test

@@ -5,7 +5,7 @@
 require sqlite_scanner
 
 statement ok
-ATTACH '__TEST_DIR__/attach_alter.db' AS s1 (TYPE SQLITE)
+ATTACH '{TEST_DIR}/attach_alter.db' AS s1 (TYPE SQLITE)
 
 # add column
 statement ok

test/sql/storage/attach_big.test_slow

@@ -5,7 +5,7 @@
 require sqlite_scanner
 
 statement ok
-ATTACH '__TEST_DIR__/attach_big.db' AS s (TYPE SQLITE)
+ATTACH '{TEST_DIR}/attach_big.db' AS s (TYPE SQLITE)
 
 statement ok
 USE s;

test/sql/storage/attach_catalog.test

@@ -5,7 +5,7 @@
 require sqlite_scanner
 
 statement ok
-ATTACH '__TEST_DIR__/attach_catalog.db' AS s (TYPE SQLITE)
+ATTACH '{TEST_DIR}/attach_catalog.db' AS s (TYPE SQLITE)
 
 # schemas
 statement error

test/sql/storage/attach_checkpoint.test

@@ -5,7 +5,7 @@
 require sqlite_scanner
 
 statement ok
-ATTACH '__TEST_DIR__/attach_checkpoint.db' AS s (TYPE SQLITE)
+ATTACH '{TEST_DIR}/attach_checkpoint.db' AS s (TYPE SQLITE)
 
 statement ok
 CHECKPOINT s

test/sql/storage/attach_concurrent_clients.test

@@ -4,10 +4,10 @@
 
 require sqlite_scanner
 
-require nowindows
+require notwindows
 
 statement ok
-ATTACH '__TEST_DIR__/attach_concurrentclients.db' AS s1 (TYPE SQLITE)
+ATTACH '{TEST_DIR}/attach_concurrentclients.db' AS s1 (TYPE SQLITE)
 
 statement ok
 CREATE TABLE s1.tbl(i INTEGER);
@@ -18,7 +18,7 @@ concurrentloop i 0 10
 loop k 0 10
 
 statement maybe
-INSERT INTO s1.tbl VALUES (${i} + ${k})
+INSERT INTO s1.tbl VALUES ({i} + {k})
 ----
 database is locked
 

test/sql/storage/attach_constraints.test

@@ -5,7 +5,7 @@
 require sqlite_scanner
 
 statement ok
-ATTACH '__TEST_DIR__/attach_constraints.db' AS s1 (TYPE SQLITE)
+ATTACH '{TEST_DIR}/attach_constraints.db' AS s1 (TYPE SQLITE)
 
 # FIXME:
 #statement ok

test/sql/storage/attach_copy_index.test

@@ -27,7 +27,7 @@ WHERE a1.name!=a2.name and a1.name='Checking';
 2018-02-20 10:59:00	Checking	250.00
 
 statement ok
-ATTACH '__TEST_DIR__/target.db' AS ddb;
+ATTACH '{TEST_DIR}/target.db' AS ddb;
 
 statement ok
 COPY FROM DATABASE s to ddb;
@@ -58,7 +58,7 @@ statement ok
 DETACH ddb
 
 statement ok
-ATTACH '__TEST_DIR__/target.db' AS ddb;
+ATTACH '{TEST_DIR}/target.db' AS ddb;
 
 statement ok
 USE ddb

test/sql/storage/attach_create_if_exists.test

@@ -5,7 +5,7 @@
 require sqlite_scanner
 
 statement ok
-ATTACH '__TEST_DIR__/attach_create_if_exists.db' AS s1 (TYPE SQLITE)
+ATTACH '{TEST_DIR}/attach_create_if_exists.db' AS s1 (TYPE SQLITE)
 
 statement ok
 CREATE TABLE s1.test(i INTEGER);

test/sql/storage/attach_create_index.test

@@ -5,7 +5,7 @@
 require sqlite_scanner
 
 statement ok
-ATTACH '__TEST_DIR__/attach_create_index.db' AS s (TYPE SQLITE)
+ATTACH '{TEST_DIR}/attach_create_index.db' AS s (TYPE SQLITE)
 
 statement ok
 CREATE TABLE s.test(i INTEGER);

test/sql/storage/attach_create_or_replace.test

@@ -5,7 +5,7 @@
 require sqlite_scanner
 
 statement ok
-ATTACH '__TEST_DIR__/attach_unnest.sqlite' AS tmp (TYPE SQLITE);
+ATTACH '{TEST_DIR}/attach_unnest.sqlite' AS tmp (TYPE SQLITE);
 
 statement ok
 USE tmp;

test/sql/storage/attach_database_list.test

@@ -5,10 +5,10 @@
 require sqlite_scanner
 
 statement ok
-ATTACH '__TEST_DIR__/attach_dblist1.db' AS s1 (TYPE SQLITE)
+ATTACH '{TEST_DIR}/attach_dblist1.db' AS s1 (TYPE SQLITE)
 
 statement ok
-ATTACH '__TEST_DIR__/attach_dblist2.db' AS s2 (TYPE SQLITE)
+ATTACH '{TEST_DIR}/attach_dblist2.db' AS s2 (TYPE SQLITE)
 
 statement ok
 SELECT * FROM duckdb_databases();

test/sql/storage/attach_defaults.test

@@ -5,7 +5,7 @@
 require sqlite_scanner
 
 statement ok
-ATTACH '__TEST_DIR__/attach_defaults.db' AS s1 (TYPE SQLITE)
+ATTACH '{TEST_DIR}/attach_defaults.db' AS s1 (TYPE SQLITE)
 
 statement ok
 CREATE TABLE s1.test(i INTEGER DEFAULT 42, j INTEGER);

test/sql/storage/attach_delete.test

@@ -5,7 +5,7 @@
 require sqlite_scanner
 
 statement ok
-ATTACH '__TEST_DIR__/attach_delete.db' AS s1 (TYPE SQLITE)
+ATTACH '{TEST_DIR}/attach_delete.db' AS s1 (TYPE SQLITE)
 
 statement ok
 CREATE TABLE s1.test(i INTEGER);