text2sql-eval-toolkit/data/test-benchmarks.json at main · IBM/text2sql-eval-toolkit · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
{
    "bird_mini_dev_sqlite_test_50": {
        "name": "bird_mini_dev_sqlite_test_50",
        "description": "Test sample of 50 BIRD-SQL Mini-Dev in SQLite https://github.com/bird-bench/mini_dev",
        "data": "benchmarks/test_benchmarks/bird_mini_dev_sqlite_test_50.json",
        "schema": "benchmarks/bird_mini_dev_sqlite-schema.json",
        "predictions": "benchmarks/test_benchmarks/results/bird_mini_dev_sqlite_test_50-predictions.json",
        "db_engine": {
            "db_type": "sqlite",
            "db_folder": "benchmarks/dbs/bird/dev_databases"
        }
    },
    "bird_mini_dev_postgres_test_50": {
        "name": "bird_mini_dev_postgres_test_50",
        "description": "Test sample of 50 BIRD-SQL Mini-Dev in PostgreSQL https://github.com/bird-bench/mini_dev",
        "data": "benchmarks/test_benchmarks/bird_mini_dev_postgres_test_50.json",
        "schema": "benchmarks/bird_mini_dev_postgres-schema.json",
        "predictions": "benchmarks/test_benchmarks/results/bird_mini_dev_postgres_test_50-predictions.json",
        "db_engine": {
            "db_type": "postgres",
            "schema_name": "public",
            "connection_string_env_var": "POSTGRES_CONNECTION_STRING"
        }
    },
    "spider_dev_test_50": {
        "name": "spider_dev_test_50",
        "description": "Test sample of 50 Spider Dev questions https://yale-lily.github.io/spider",
        "data": "benchmarks/test_benchmarks/spider_dev_test_50.json",
        "schema": "benchmarks/spider-dev-schema.json",
        "predictions": "benchmarks/test_benchmarks/results/spider_dev_test_50-predictions.json",
        "db_engine": {
            "db_type": "sqlite",
            "db_folder": "benchmarks/dbs/spider/database"
        }
    },
    "beaver_test_10": {
        "name": "beaver_test_10",
        "description": "Test sample of 10 Beaver benchmark questions https://peterbaile.github.io/beaver/",
        "data": "benchmarks/test_benchmarks/beaver_test_10.json",
        "schema": "benchmarks/beaver-schema.json",
        "predictions": "benchmarks/test_benchmarks/results/beaver_test_10-predictions.json",
        "db_engine": {
            "db_type": "mysql",
            "connection_string_env_var": "MYSQL_CONNECTION_STRING"
        }
    },
    "archer_en_dev_test_10": {
        "name": "archer_en_dev_test_10",
        "description": "Test sample of 10 Archer English Dev questions https://sig4kg.github.io/archer-bench/",
        "data": "benchmarks/test_benchmarks/archer_en_dev_test_10.json",
        "schema": "benchmarks/archer-schema.json",
        "predictions": "benchmarks/test_benchmarks/results/archer_en_dev_test_10-predictions.json",
        "db_engine": {
            "db_type": "sqlite",
            "db_folder": "benchmarks/dbs/archer/database"
        }
    },
    "bird_sqlite_test_benchmark": {
        "name": "bird_sqlite_test_benchmark",
        "description": "3 records from BIRD mini-dev used for testing this code base",
        "data": "benchmarks/test_benchmarks/bird_sqlite_test_benchmark.json",
        "schema": "benchmarks/test_benchmarks/bird_sqlite_test_benchmark-schema.json",
        "predictions": "benchmarks/test_benchmarks/results/bird_sqlite_test_benchmark-predictions.json",
        "db_engine": {
            "db_type": "sqlite",
            "db_folder": "benchmarks/test_benchmarks/db"
        }
    }
}