Skip to content

Commit 372652f

Browse files
theorytheory
committed
Add JSON support for the HTTP engine
Map it to the `JSON` data type and test table importing, insertion, and selection. Add a test for both the binary and HTTP engines, and tweak the ClickHouse configuration to enable the JSON type in 24 and earlier, although it does not work prior to 24.8. The Binary engine will successfully import a table with a `JSON` column but not be able to insert or select that column until `JSON` support is added to `clickhouse-cpp` (issue ClickHouse/clickhouse-cpp#422; PR ClickHouse/clickhouse-cpp#450).
1 parent 6ec5cff commit 372652f

File tree

7 files changed

+312
-0
lines changed

7 files changed

+312
-0
lines changed

‎.github/ubuntu/clickhouse.sh‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,5 +38,17 @@ for pkg in clickhouse-common-static clickhouse-server; do
3838
rm "${pkg}.deb"
3939
done
4040

41+
if [ "${CH_VERSION%%.*}" -lt 25 ]; then
42+
# Enable the JSON type in release prior to 2025.
43+
printf "~~~~ Starting ClickHouse %s ~~~~\n" "$CH_VERSION"
44+
setting=allow_experimental_object_type
45+
if [ "${CH_VERSION:0:4}" == "24.8" ]; then
46+
# The setting was renamed.
47+
setting=allow_experimental_json_type
48+
fi
49+
50+
perl -i -pe "s{^((\s+)<default>)}{\$1\n\$2 <$setting>1</$setting>}" /etc/clickhouse-server/users.xml
51+
fi
52+
4153
printf "~~~~ Starting ClickHouse %s ~~~~\n" "$CH_VERSION"
4254
/etc/init.d/clickhouse-server start

‎src/pglink.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -760,6 +760,7 @@ static char *str_types_map[][2] = {
760760
{"UUID", "UUID"},
761761
{"IPv4", "inet"},
762762
{"IPv6", "inet"},
763+
{"JSON", "JSON"},
763764
{NULL, NULL},
764765
};
765766

‎test/expected/json.out‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
SET datestyle = 'ISO';
2+
CREATE SERVER binary_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'binary');
3+
CREATE SERVER http_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'http');
4+
CREATE USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback;
5+
CREATE USER MAPPING FOR CURRENT_USER SERVER http_json_loopback;
6+
SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS json_test');
7+
clickhouse_raw_query
8+
----------------------
9+
10+
(1 row)
11+
12+
SELECT clickhouse_raw_query('CREATE DATABASE json_test');
13+
clickhouse_raw_query
14+
----------------------
15+
16+
(1 row)
17+
18+
SELECT clickhouse_raw_query($$
19+
CREATE TABLE json_test.things (
20+
id Int32 NOT NULL,
21+
data JSON NOT NULL
22+
) ENGINE = MergeTree PARTITION BY id ORDER BY (id);
23+
$$);
24+
clickhouse_raw_query
25+
----------------------
26+
27+
(1 row)
28+
29+
CREATE SCHEMA json_bin;
30+
CREATE SCHEMA json_http;
31+
IMPORT FOREIGN SCHEMA "json_test" FROM SERVER binary_json_loopback INTO json_bin;
32+
\d json_bin.things
33+
Foreign table "json_bin.things"
34+
Column | Type | Collation | Nullable | Default | FDW options
35+
--------+---------+-----------+----------+---------+-------------
36+
id | integer | | not null | |
37+
data | json | | not null | |
38+
Server: binary_json_loopback
39+
FDW options: (database 'json_test', table_name 'things', engine 'MergeTree')
40+
41+
IMPORT FOREIGN SCHEMA "json_test" FROM SERVER http_json_loopback INTO json_http;
42+
\d json_http.things
43+
Foreign table "json_http.things"
44+
Column | Type | Collation | Nullable | Default | FDW options
45+
--------+---------+-----------+----------+---------+-------------
46+
id | integer | | not null | |
47+
data | json | | not null | |
48+
Server: http_json_loopback
49+
FDW options: (database 'json_test', table_name 'things', engine 'MergeTree')
50+
51+
-- Fails pending https://github.com/ClickHouse/clickhouse-cpp/issues/422
52+
INSERT INTO json_bin.things VALUES
53+
(1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'),
54+
(2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}')
55+
;
56+
ERROR: pg_clickhouse: could not prepare insert - unsupported column type: JSON
57+
INSERT INTO json_http.things VALUES
58+
(1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'),
59+
(2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}'),
60+
(3, '{"id": 3, "name": "gizmo", "size": "medium", "stocked": true}'),
61+
(4, '{"id": 4, "name": "doodad", "size": "large", "stocked": false}')
62+
;
63+
SELECT * FROM json_bin.things ORDER BY id;
64+
ERROR: pg_clickhouse: unsupported column type: JSON
65+
DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC
66+
SELECT * FROM json_http.things ORDER BY id;
67+
id | data
68+
----+----------------------------------------------------------
69+
1 | {"id":1,"name":"widget","size":"large","stocked":true}
70+
2 | {"id":2,"name":"sprocket","size":"small","stocked":true}
71+
3 | {"id":3,"name":"gizmo","size":"medium","stocked":true}
72+
4 | {"id":4,"name":"doodad","size":"large","stocked":false}
73+
(4 rows)
74+
75+
SELECT clickhouse_raw_query('DROP DATABASE json_test');
76+
clickhouse_raw_query
77+
----------------------
78+
79+
(1 row)
80+
81+
DROP USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback;
82+
DROP USER MAPPING FOR CURRENT_USER SERVER http_json_loopback;
83+
DROP SERVER binary_json_loopback CASCADE;
84+
NOTICE: drop cascades to foreign table json_bin.things
85+
DROP SERVER http_json_loopback CASCADE;
86+
NOTICE: drop cascades to foreign table json_http.things

‎test/expected/json_1.out‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
SET datestyle = 'ISO';
2+
CREATE SERVER binary_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'binary');
3+
CREATE SERVER http_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'http');
4+
CREATE USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback;
5+
CREATE USER MAPPING FOR CURRENT_USER SERVER http_json_loopback;
6+
SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS json_test');
7+
clickhouse_raw_query
8+
----------------------
9+
10+
(1 row)
11+
12+
SELECT clickhouse_raw_query('CREATE DATABASE json_test');
13+
clickhouse_raw_query
14+
----------------------
15+
16+
(1 row)
17+
18+
SELECT clickhouse_raw_query($$
19+
CREATE TABLE json_test.things (
20+
id Int32 NOT NULL,
21+
data JSON NOT NULL
22+
) ENGINE = MergeTree PARTITION BY id ORDER BY (id);
23+
$$);
24+
clickhouse_raw_query
25+
----------------------
26+
27+
(1 row)
28+
29+
CREATE SCHEMA json_bin;
30+
CREATE SCHEMA json_http;
31+
IMPORT FOREIGN SCHEMA "json_test" FROM SERVER binary_json_loopback INTO json_bin;
32+
\d json_bin.things
33+
Foreign table "json_bin.things"
34+
Column | Type | Collation | Nullable | Default | FDW options
35+
--------+---------+-----------+----------+---------+-------------
36+
id | integer | | not null | |
37+
data | json | | not null | |
38+
Server: binary_json_loopback
39+
FDW options: (database 'json_test', table_name 'things', engine 'MergeTree')
40+
41+
IMPORT FOREIGN SCHEMA "json_test" FROM SERVER http_json_loopback INTO json_http;
42+
\d json_http.things
43+
Foreign table "json_http.things"
44+
Column | Type | Collation | Nullable | Default | FDW options
45+
--------+---------+-----------+----------+---------+-------------
46+
id | integer | | not null | |
47+
data | json | | not null | |
48+
Server: http_json_loopback
49+
FDW options: (database 'json_test', table_name 'things', engine 'MergeTree')
50+
51+
-- Fails pending https://github.com/ClickHouse/clickhouse-cpp/issues/422
52+
INSERT INTO json_bin.things VALUES
53+
(1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'),
54+
(2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}')
55+
;
56+
ERROR: pg_clickhouse: could not prepare insert - unsupported column type: JSON
57+
INSERT INTO json_http.things VALUES
58+
(1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'),
59+
(2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}'),
60+
(3, '{"id": 3, "name": "gizmo", "size": "medium", "stocked": true}'),
61+
(4, '{"id": 4, "name": "doodad", "size": "large", "stocked": false}')
62+
;
63+
SELECT * FROM json_bin.things ORDER BY id;
64+
ERROR: pg_clickhouse: unsupported column type: JSON
65+
DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC
66+
SELECT * FROM json_http.things ORDER BY id;
67+
id | data
68+
----+------------------------------------------------------------
69+
1 | {"id":"1","name":"widget","size":"large","stocked":true}
70+
2 | {"id":"2","name":"sprocket","size":"small","stocked":true}
71+
3 | {"id":"3","name":"gizmo","size":"medium","stocked":true}
72+
4 | {"id":"4","name":"doodad","size":"large","stocked":false}
73+
(4 rows)
74+
75+
SELECT clickhouse_raw_query('DROP DATABASE json_test');
76+
clickhouse_raw_query
77+
----------------------
78+
79+
(1 row)
80+
81+
DROP USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback;
82+
DROP USER MAPPING FOR CURRENT_USER SERVER http_json_loopback;
83+
DROP SERVER binary_json_loopback CASCADE;
84+
NOTICE: drop cascades to foreign table json_bin.things
85+
DROP SERVER http_json_loopback CASCADE;
86+
NOTICE: drop cascades to foreign table json_http.things

‎test/expected/json_2.out‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
SET datestyle = 'ISO';
2+
CREATE SERVER binary_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'binary');
3+
CREATE SERVER http_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'http');
4+
CREATE USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback;
5+
CREATE USER MAPPING FOR CURRENT_USER SERVER http_json_loopback;
6+
SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS json_test');
7+
clickhouse_raw_query
8+
----------------------
9+
10+
(1 row)
11+
12+
SELECT clickhouse_raw_query('CREATE DATABASE json_test');
13+
clickhouse_raw_query
14+
----------------------
15+
16+
(1 row)
17+
18+
SELECT clickhouse_raw_query($$
19+
CREATE TABLE json_test.things (
20+
id Int32 NOT NULL,
21+
data JSON NOT NULL
22+
) ENGINE = MergeTree PARTITION BY id ORDER BY (id);
23+
$$);
24+
clickhouse_raw_query
25+
----------------------
26+
27+
(1 row)
28+
29+
CREATE SCHEMA json_bin;
30+
CREATE SCHEMA json_http;
31+
IMPORT FOREIGN SCHEMA "json_test" FROM SERVER binary_json_loopback INTO json_bin;
32+
ERROR: pg_clickhouse: could not map things.data type <'json')>
33+
\d json_bin.things
34+
IMPORT FOREIGN SCHEMA "json_test" FROM SERVER http_json_loopback INTO json_http;
35+
ERROR: pg_clickhouse: could not map things.data type <'json')>
36+
\d json_http.things
37+
-- Fails pending https://github.com/ClickHouse/clickhouse-cpp/issues/422
38+
INSERT INTO json_bin.things VALUES
39+
(1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'),
40+
(2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}')
41+
;
42+
ERROR: relation "json_bin.things" does not exist
43+
LINE 1: INSERT INTO json_bin.things VALUES
44+
^
45+
INSERT INTO json_http.things VALUES
46+
(1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'),
47+
(2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}'),
48+
(3, '{"id": 3, "name": "gizmo", "size": "medium", "stocked": true}'),
49+
(4, '{"id": 4, "name": "doodad", "size": "large", "stocked": false}')
50+
;
51+
ERROR: relation "json_http.things" does not exist
52+
LINE 1: INSERT INTO json_http.things VALUES
53+
^
54+
SELECT * FROM json_bin.things ORDER BY id;
55+
ERROR: relation "json_bin.things" does not exist
56+
LINE 1: SELECT * FROM json_bin.things ORDER BY id;
57+
^
58+
SELECT * FROM json_http.things ORDER BY id;
59+
ERROR: relation "json_http.things" does not exist
60+
LINE 1: SELECT * FROM json_http.things ORDER BY id;
61+
^
62+
SELECT clickhouse_raw_query('DROP DATABASE json_test');
63+
clickhouse_raw_query
64+
----------------------
65+
66+
(1 row)
67+
68+
DROP USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback;
69+
DROP USER MAPPING FOR CURRENT_USER SERVER http_json_loopback;
70+
DROP SERVER binary_json_loopback CASCADE;
71+
DROP SERVER http_json_loopback CASCADE;

‎test/expected/result_map.txt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,3 +100,16 @@ import_schema.sql
100100
ClickHouse | File
101101
------------|-------------------
102102
22-25 | import_schema.out
103+
104+
json.sql
105+
--------
106+
107+
Postgres | File
108+
----------|----------
109+
13-18 | json.out
110+
111+
ClickHouse | File
112+
------------|------------
113+
25.8+ | json.out
114+
24.8-25.3 | json_1.out
115+
22-24.3 | json_2.out

‎test/sql/json.sql‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
SET datestyle = 'ISO';
2+
CREATE SERVER binary_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'binary');
3+
CREATE SERVER http_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'http');
4+
CREATE USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback;
5+
CREATE USER MAPPING FOR CURRENT_USER SERVER http_json_loopback;
6+
7+
SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS json_test');
8+
SELECT clickhouse_raw_query('CREATE DATABASE json_test');
9+
SELECT clickhouse_raw_query($$
10+
CREATE TABLE json_test.things (
11+
id Int32 NOT NULL,
12+
data JSON NOT NULL
13+
) ENGINE = MergeTree PARTITION BY id ORDER BY (id);
14+
$$);
15+
16+
CREATE SCHEMA json_bin;
17+
CREATE SCHEMA json_http;
18+
IMPORT FOREIGN SCHEMA "json_test" FROM SERVER binary_json_loopback INTO json_bin;
19+
\d json_bin.things
20+
IMPORT FOREIGN SCHEMA "json_test" FROM SERVER http_json_loopback INTO json_http;
21+
\d json_http.things
22+
23+
-- Fails pending https://github.com/ClickHouse/clickhouse-cpp/issues/422
24+
INSERT INTO json_bin.things VALUES
25+
(1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'),
26+
(2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}')
27+
;
28+
29+
INSERT INTO json_http.things VALUES
30+
(1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'),
31+
(2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}'),
32+
(3, '{"id": 3, "name": "gizmo", "size": "medium", "stocked": true}'),
33+
(4, '{"id": 4, "name": "doodad", "size": "large", "stocked": false}')
34+
;
35+
36+
SELECT * FROM json_bin.things ORDER BY id;
37+
SELECT * FROM json_http.things ORDER BY id;
38+
39+
SELECT clickhouse_raw_query('DROP DATABASE json_test');
40+
DROP USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback;
41+
DROP USER MAPPING FOR CURRENT_USER SERVER http_json_loopback;
42+
DROP SERVER binary_json_loopback CASCADE;
43+
DROP SERVER http_json_loopback CASCADE;

0 commit comments

Comments
 (0)