text2sql-eval-toolkit/data/benchmarks.json at main · IBM/text2sql-eval-toolkit · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
{
    "bird_mini_dev_sqlite": {
        "name": "bird_mini_dev_sqlite",
        "description": "BIRD-SQL Mini-Dev in SQLite https://github.com/bird-bench/mini_dev",
        "data": "benchmarks/bird_mini_dev_sqlite.json",
        "schema": "benchmarks/bird_mini_dev_sqlite-schema.json",
        "predictions": "results/bird_mini_dev_sqlite-predictions.json",
        "db_engine": {
            "db_type": "sqlite",
            "db_folder": "benchmarks/dbs/bird/dev_databases",
            "schema_name": "",
            "connection_string_env_var": ""
        },
        "logo": "bird_mini_dev_sqlite.png"
    },
    "bird_mini_dev_postgres": {
        "name": "bird_mini_dev_postgres",
        "description": "BIRD-SQL Mini-Dev in PostgreSQL https://github.com/bird-bench/mini_dev",
        "data": "benchmarks/bird_mini_dev_postgres.json",
        "schema": "benchmarks/bird_mini_dev_postgres-schema.json",
        "predictions": "results/bird_mini_dev_postgres-predictions.json",
        "db_engine": {
            "db_type": "postgres",
            "db_folder": "",
            "schema_name": "public",
            "connection_string_env_var": "POSTGRES_CONNECTION_STRING"
        },
        "logo": "bird_mini_dev_postgres.png"
    },
    "beaver": {
        "name": "beaver",
        "description": "Beaver benchmark https://peterbaile.github.io/beaver/",
        "data": "benchmarks/beaver.json",
        "schema": "benchmarks/beaver-schema.json",
        "predictions": "results/beaver-predictions.json",
        "db_engine": {
            "db_type": "mysql",
            "db_folder": "",
            "schema_name": "",
            "connection_string_env_var": "MYSQL_CONNECTION_STRING"
        },
        "logo": "beaver.png"
    },
    "archer_en_dev": {
        "name": "archer_en_dev",
        "description": "Archer English Dev Set https://sig4kg.github.io/archer-bench/",
        "data": "benchmarks/archer_en_dev.json",
        "schema": "benchmarks/archer-schema.json",
        "predictions": "results/archer_en_dev-predictions.json",
        "db_engine": {
            "db_type": "sqlite",
            "db_folder": "benchmarks/dbs/archer/database",
            "schema_name": "",
            "connection_string_env_var": ""
        },
        "logo": "archer_en_dev.png"
    },
    "spider_dev": {
        "name": "spider_dev",
        "description": "Spider Dev Set - Full 1,034 questions https://yale-lily.github.io/spider",
        "data": "benchmarks/spider-dev-converted.json",
        "schema": "benchmarks/spider-dev-schema.json",
        "predictions": "results/spider_dev-predictions.json",
        "db_engine": {
            "db_type": "sqlite",
            "db_folder": "benchmarks/dbs/spider/database",
            "schema_name": "",
            "connection_string_env_var": ""
        },
        "logo": "spider_dev.png"
    },
    "spider_realistic": {
        "name": "spider_realistic",
        "description": "Spider Realistic Dataset In Structure-Grounded Pretraining for Text-to-SQL https://zenodo.org/records/5205322",
        "data": "benchmarks/spider-realistic.json",
        "schema": "benchmarks/spider-dev-schema.json",
        "predictions": "results/spider_realistic-predictions.json",
        "db_engine": {
            "db_type": "sqlite",
            "db_folder": "benchmarks/dbs/spider/database",
            "schema_name": "",
            "connection_string_env_var": ""
        },
        "logo": "spider_realistic.png"
    }
}