Skip to content

Commit 66d5f50

Browse files
theorytheory
committed
Add JSON support for the HTTP engine
Map it to the `JSON` data type and test table importing, insertion, and selection. Add a test for both the binary and HTTP engines, and tweak the ClickHouse configuration to enable the JSON type in 24 and earlier, although it does not work prior to 24.8. The Binary engine will successfully import a table with a `JSON` column but not be able to insert or select that column until `JSON` support is added to `clickhouse-cpp` (issue ClickHouse/clickhouse-cpp#422; PR ClickHouse/clickhouse-cpp#450).
1 parent 6ec5cff commit 66d5f50

File tree

7 files changed

+312
-0
lines changed

7 files changed

+312
-0
lines changed

‎.github/ubuntu/clickhouse.sh‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,5 +38,17 @@ for pkg in clickhouse-common-static clickhouse-server; do
3838
rm "${pkg}.deb"
3939
done
4040

41+
if [ "${CH_VERSION%%.*}" -lt 25 ]; then
42+
# Enable the JSON type in release prior to 2025.
43+
printf "~~~~ Starting ClickHouse %s ~~~~\n" "$CH_VERSION"
44+
setting=allow_experimental_object_type
45+
if [ "${CH_VERSION:0:4}" == "24.8" ]; then
46+
# The setting was renamed.
47+
setting=allow_experimental_json_type
48+
fi
49+
50+
perl -i -pe "s{^((\s+)<default>)}{\$1\n\$2 <$setting>1</$setting>}" /etc/clickhouse-server/users.xml
51+
fi
52+
4153
printf "~~~~ Starting ClickHouse %s ~~~~\n" "$CH_VERSION"
4254
/etc/init.d/clickhouse-server start

‎src/pglink.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -760,6 +760,7 @@ static char *str_types_map[][2] = {
760760
{"UUID", "UUID"},
761761
{"IPv4", "inet"},
762762
{"IPv6", "inet"},
763+
{"JSON", "JSON"},
763764
{NULL, NULL},
764765
};
765766

‎test/expected/json.out‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
SET datestyle = 'ISO';
2+
CREATE SERVER binary_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'binary');
3+
CREATE SERVER http_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'http');
4+
CREATE USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback;
5+
CREATE USER MAPPING FOR CURRENT_USER SERVER http_json_loopback;
6+
SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS json_test');
7+
clickhouse_raw_query
8+
----------------------
9+
10+
(1 row)
11+
12+
SELECT clickhouse_raw_query('CREATE DATABASE json_test');
13+
clickhouse_raw_query
14+
----------------------
15+
16+
(1 row)
17+
18+
SELECT clickhouse_raw_query($$
19+
CREATE TABLE json_test.things (
20+
id Int32 NOT NULL,
21+
data JSON NOT NULL
22+
) ENGINE = MergeTree PARTITION BY id ORDER BY (id);
23+
$$);
24+
clickhouse_raw_query
25+
----------------------
26+
27+
(1 row)
28+
29+
CREATE SCHEMA bin;
30+
CREATE SCHEMA http;
31+
IMPORT FOREIGN SCHEMA "json_test" FROM SERVER binary_json_loopback INTO bin;
32+
\d bin.things
33+
Foreign table "bin.things"
34+
Column | Type | Collation | Nullable | Default | FDW options
35+
--------+---------+-----------+----------+---------+-------------
36+
id | integer | | not null | |
37+
data | json | | not null | |
38+
Server: binary_json_loopback
39+
FDW options: (database 'json_test', table_name 'things', engine 'MergeTree')
40+
41+
IMPORT FOREIGN SCHEMA "json_test" FROM SERVER http_json_loopback INTO http;
42+
\d http.things
43+
Foreign table "http.things"
44+
Column | Type | Collation | Nullable | Default | FDW options
45+
--------+---------+-----------+----------+---------+-------------
46+
id | integer | | not null | |
47+
data | json | | not null | |
48+
Server: http_json_loopback
49+
FDW options: (database 'json_test', table_name 'things', engine 'MergeTree')
50+
51+
-- Fails pending https://github.com/ClickHouse/clickhouse-cpp/issues/422
52+
INSERT INTO bin.things VALUES
53+
(1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'),
54+
(2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}')
55+
;
56+
ERROR: pg_clickhouse: could not prepare insert - unsupported column type: JSON
57+
INSERT INTO http.things VALUES
58+
(1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'),
59+
(2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}'),
60+
(3, '{"id": 3, "name": "gizmo", "size": "medium", "stocked": true}'),
61+
(4, '{"id": 4, "name": "doodad", "size": "large", "stocked": false}')
62+
;
63+
SELECT * FROM bin.things ORDER BY id;
64+
ERROR: pg_clickhouse: unsupported column type: JSON
65+
DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC
66+
SELECT * FROM http.things ORDER BY id;
67+
id | data
68+
----+----------------------------------------------------------
69+
1 | {"id":1,"name":"widget","size":"large","stocked":true}
70+
2 | {"id":2,"name":"sprocket","size":"small","stocked":true}
71+
3 | {"id":3,"name":"gizmo","size":"medium","stocked":true}
72+
4 | {"id":4,"name":"doodad","size":"large","stocked":false}
73+
(4 rows)
74+
75+
SELECT clickhouse_raw_query('DROP DATABASE json_test');
76+
clickhouse_raw_query
77+
----------------------
78+
79+
(1 row)
80+
81+
DROP USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback;
82+
DROP USER MAPPING FOR CURRENT_USER SERVER http_json_loopback;
83+
DROP SERVER binary_json_loopback CASCADE;
84+
NOTICE: drop cascades to foreign table bin.things
85+
DROP SERVER http_json_loopback CASCADE;
86+
NOTICE: drop cascades to foreign table http.things

‎test/expected/json_1.out‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
SET datestyle = 'ISO';
2+
CREATE SERVER binary_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'binary');
3+
CREATE SERVER http_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'http');
4+
CREATE USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback;
5+
CREATE USER MAPPING FOR CURRENT_USER SERVER http_json_loopback;
6+
SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS json_test');
7+
clickhouse_raw_query
8+
----------------------
9+
10+
(1 row)
11+
12+
SELECT clickhouse_raw_query('CREATE DATABASE json_test');
13+
clickhouse_raw_query
14+
----------------------
15+
16+
(1 row)
17+
18+
SELECT clickhouse_raw_query($$
19+
CREATE TABLE json_test.things (
20+
id Int32 NOT NULL,
21+
data JSON NOT NULL
22+
) ENGINE = MergeTree PARTITION BY id ORDER BY (id);
23+
$$);
24+
clickhouse_raw_query
25+
----------------------
26+
27+
(1 row)
28+
29+
CREATE SCHEMA bin;
30+
CREATE SCHEMA http;
31+
IMPORT FOREIGN SCHEMA "json_test" FROM SERVER binary_json_loopback INTO bin;
32+
\d bin.things
33+
Foreign table "bin.things"
34+
Column | Type | Collation | Nullable | Default | FDW options
35+
--------+---------+-----------+----------+---------+-------------
36+
id | integer | | not null | |
37+
data | json | | not null | |
38+
Server: binary_json_loopback
39+
FDW options: (database 'json_test', table_name 'things', engine 'MergeTree')
40+
41+
IMPORT FOREIGN SCHEMA "json_test" FROM SERVER http_json_loopback INTO http;
42+
\d http.things
43+
Foreign table "http.things"
44+
Column | Type | Collation | Nullable | Default | FDW options
45+
--------+---------+-----------+----------+---------+-------------
46+
id | integer | | not null | |
47+
data | json | | not null | |
48+
Server: http_json_loopback
49+
FDW options: (database 'json_test', table_name 'things', engine 'MergeTree')
50+
51+
-- Fails pending https://github.com/ClickHouse/clickhouse-cpp/issues/422
52+
INSERT INTO bin.things VALUES
53+
(1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'),
54+
(2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}')
55+
;
56+
ERROR: pg_clickhouse: could not prepare insert - unsupported column type: JSON
57+
INSERT INTO http.things VALUES
58+
(1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'),
59+
(2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}'),
60+
(3, '{"id": 3, "name": "gizmo", "size": "medium", "stocked": true}'),
61+
(4, '{"id": 4, "name": "doodad", "size": "large", "stocked": false}')
62+
;
63+
SELECT * FROM bin.things ORDER BY id;
64+
ERROR: pg_clickhouse: unsupported column type: JSON
65+
DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC
66+
SELECT * FROM http.things ORDER BY id;
67+
id | data
68+
----+------------------------------------------------------------
69+
1 | {"id":"1","name":"widget","size":"large","stocked":true}
70+
2 | {"id":"2","name":"sprocket","size":"small","stocked":true}
71+
3 | {"id":"3","name":"gizmo","size":"medium","stocked":true}
72+
4 | {"id":"4","name":"doodad","size":"large","stocked":false}
73+
(4 rows)
74+
75+
SELECT clickhouse_raw_query('DROP DATABASE json_test');
76+
clickhouse_raw_query
77+
----------------------
78+
79+
(1 row)
80+
81+
DROP USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback;
82+
DROP USER MAPPING FOR CURRENT_USER SERVER http_json_loopback;
83+
DROP SERVER binary_json_loopback CASCADE;
84+
NOTICE: drop cascades to foreign table bin.things
85+
DROP SERVER http_json_loopback CASCADE;
86+
NOTICE: drop cascades to foreign table http.things

‎test/expected/json_2.out‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
SET datestyle = 'ISO';
2+
CREATE SERVER binary_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'binary');
3+
CREATE SERVER http_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'http');
4+
CREATE USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback;
5+
CREATE USER MAPPING FOR CURRENT_USER SERVER http_json_loopback;
6+
SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS json_test');
7+
clickhouse_raw_query
8+
----------------------
9+
10+
(1 row)
11+
12+
SELECT clickhouse_raw_query('CREATE DATABASE json_test');
13+
clickhouse_raw_query
14+
----------------------
15+
16+
(1 row)
17+
18+
SELECT clickhouse_raw_query($$
19+
CREATE TABLE json_test.things (
20+
id Int32 NOT NULL,
21+
data JSON NOT NULL
22+
) ENGINE = MergeTree PARTITION BY id ORDER BY (id);
23+
$$);
24+
clickhouse_raw_query
25+
----------------------
26+
27+
(1 row)
28+
29+
CREATE SCHEMA bin;
30+
CREATE SCHEMA http;
31+
IMPORT FOREIGN SCHEMA "json_test" FROM SERVER binary_json_loopback INTO bin;
32+
ERROR: pg_clickhouse: could not map things.data type <'json')>
33+
\d bin.things
34+
IMPORT FOREIGN SCHEMA "json_test" FROM SERVER http_json_loopback INTO http;
35+
ERROR: pg_clickhouse: could not map things.data type <'json')>
36+
\d http.things
37+
-- Fails pending https://github.com/ClickHouse/clickhouse-cpp/issues/422
38+
INSERT INTO bin.things VALUES
39+
(1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'),
40+
(2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}')
41+
;
42+
ERROR: relation "bin.things" does not exist
43+
LINE 1: INSERT INTO bin.things VALUES
44+
^
45+
INSERT INTO http.things VALUES
46+
(1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'),
47+
(2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}'),
48+
(3, '{"id": 3, "name": "gizmo", "size": "medium", "stocked": true}'),
49+
(4, '{"id": 4, "name": "doodad", "size": "large", "stocked": false}')
50+
;
51+
ERROR: relation "http.things" does not exist
52+
LINE 1: INSERT INTO http.things VALUES
53+
^
54+
SELECT * FROM bin.things ORDER BY id;
55+
ERROR: relation "bin.things" does not exist
56+
LINE 1: SELECT * FROM bin.things ORDER BY id;
57+
^
58+
SELECT * FROM http.things ORDER BY id;
59+
ERROR: relation "http.things" does not exist
60+
LINE 1: SELECT * FROM http.things ORDER BY id;
61+
^
62+
SELECT clickhouse_raw_query('DROP DATABASE json_test');
63+
clickhouse_raw_query
64+
----------------------
65+
66+
(1 row)
67+
68+
DROP USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback;
69+
DROP USER MAPPING FOR CURRENT_USER SERVER http_json_loopback;
70+
DROP SERVER binary_json_loopback CASCADE;
71+
DROP SERVER http_json_loopback CASCADE;

‎test/expected/result_map.txt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,3 +100,16 @@ import_schema.sql
100100
ClickHouse | File
101101
------------|-------------------
102102
22-25 | import_schema.out
103+
104+
json.sql
105+
--------
106+
107+
Postgres | File
108+
----------|----------
109+
13-18 | json.out
110+
111+
ClickHouse | File
112+
------------|------------
113+
25.8+ | json.out
114+
24.8-25.3 | json_1.out
115+
22-24.3 | json_2.out

‎test/sql/json.sql‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
SET datestyle = 'ISO';
2+
CREATE SERVER binary_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'binary');
3+
CREATE SERVER http_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'http');
4+
CREATE USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback;
5+
CREATE USER MAPPING FOR CURRENT_USER SERVER http_json_loopback;
6+
7+
SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS json_test');
8+
SELECT clickhouse_raw_query('CREATE DATABASE json_test');
9+
SELECT clickhouse_raw_query($$
10+
CREATE TABLE json_test.things (
11+
id Int32 NOT NULL,
12+
data JSON NOT NULL
13+
) ENGINE = MergeTree PARTITION BY id ORDER BY (id);
14+
$$);
15+
16+
CREATE SCHEMA bin;
17+
CREATE SCHEMA http;
18+
IMPORT FOREIGN SCHEMA "json_test" FROM SERVER binary_json_loopback INTO bin;
19+
\d bin.things
20+
IMPORT FOREIGN SCHEMA "json_test" FROM SERVER http_json_loopback INTO http;
21+
\d http.things
22+
23+
-- Fails pending https://github.com/ClickHouse/clickhouse-cpp/issues/422
24+
INSERT INTO bin.things VALUES
25+
(1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'),
26+
(2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}')
27+
;
28+
29+
INSERT INTO http.things VALUES
30+
(1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'),
31+
(2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}'),
32+
(3, '{"id": 3, "name": "gizmo", "size": "medium", "stocked": true}'),
33+
(4, '{"id": 4, "name": "doodad", "size": "large", "stocked": false}')
34+
;
35+
36+
SELECT * FROM bin.things ORDER BY id;
37+
SELECT * FROM http.things ORDER BY id;
38+
39+
SELECT clickhouse_raw_query('DROP DATABASE json_test');
40+
DROP USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback;
41+
DROP USER MAPPING FOR CURRENT_USER SERVER http_json_loopback;
42+
DROP SERVER binary_json_loopback CASCADE;
43+
DROP SERVER http_json_loopback CASCADE;

0 commit comments

Comments
 (0)