SET datestyle = 'ISO'; CREATE SERVER binary_inserts_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'binary_inserts_test', driver 'binary'); CREATE USER MAPPING FOR CURRENT_USER SERVER binary_inserts_loopback; SELECT clickhouse_raw_query('drop database if exists binary_inserts_test'); clickhouse_raw_query ---------------------- (1 row) SELECT clickhouse_raw_query('create database binary_inserts_test'); clickhouse_raw_query ---------------------- (1 row) SELECT clickhouse_raw_query('CREATE TABLE binary_inserts_test.ints ( c1 Int8, c2 Int16, c3 Int32, c4 Int64 ) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); '); clickhouse_raw_query ---------------------- (1 row) SELECT clickhouse_raw_query('CREATE TABLE binary_inserts_test.uints ( c1 UInt8, c2 UInt16, c3 UInt32, c4 UInt64, c5 Bool ) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); '); clickhouse_raw_query ---------------------- (1 row) SELECT clickhouse_raw_query('CREATE TABLE binary_inserts_test.floats ( c1 Float32, c2 Float64 ) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1) SETTINGS allow_floating_point_partition_key=1; '); clickhouse_raw_query ---------------------- (1 row) SELECT clickhouse_raw_query('CREATE TABLE binary_inserts_test.null_ints ( c1 Int8, c2 Nullable(Int32) ) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); '); clickhouse_raw_query ---------------------- (1 row) SELECT clickhouse_raw_query('CREATE TABLE binary_inserts_test.complex ( c1 Int32, c2 Date, c3 DateTime, c4 String, c5 FixedString(10), c6 LowCardinality(String), c7 Date32, c8 DateTime64(3) ) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); '); clickhouse_raw_query ---------------------- (1 row) SELECT clickhouse_raw_query('CREATE TABLE binary_inserts_test.arrays ( c1 Int32, c2 Array(Int32), c3 Array(String) ) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); '); clickhouse_raw_query ---------------------- (1 row) SELECT clickhouse_raw_query('CREATE TABLE binary_inserts_test.addr ( c1 UUID, c2 IPv4, c3 IPv6 ) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); '); clickhouse_raw_query ---------------------- (1 row) SELECT clickhouse_raw_query('CREATE TABLE binary_inserts_test.bytes ( c1 Int8, c2 String, c3 FixedString(16) ) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1);'); clickhouse_raw_query ---------------------- (1 row) SELECT clickhouse_raw_query('CREATE TABLE binary_inserts_test.not_nullable ( c1 Int8, c2 Nullable(Enum(''x''=1)), c3 Nullable(Enum16(''x''=1)), c4 LowCardinality(Nullable(String)) ) ENGINE = MergeTree ORDER BY (c1);'); clickhouse_raw_query ---------------------- (1 row) CREATE SCHEMA binary_inserts_test; IMPORT FOREIGN SCHEMA binary_inserts_test FROM SERVER binary_inserts_loopback INTO binary_inserts_test; SET search_path = binary_inserts_test, public; /* ints */ INSERT INTO ints SELECT i, i + 1, i + 2, i+ 3 FROM generate_series(1, 3) i; SELECT * FROM ints ORDER BY c1; c1 | c2 | c3 | c4 ----+----+----+---- 1 | 2 | 3 | 4 2 | 3 | 4 | 5 3 | 4 | 5 | 6 (3 rows) INSERT INTO ints (c1, c4, c3, c2) SELECT i, i + 1, i + 2, i+ 3 FROM generate_series(4, 6) i; SELECT * FROM ints ORDER BY c1; c1 | c2 | c3 | c4 ----+----+----+---- 1 | 2 | 3 | 4 2 | 3 | 4 | 5 3 | 4 | 5 | 6 4 | 7 | 6 | 5 5 | 8 | 7 | 6 6 | 9 | 8 | 7 (6 rows) /* check dropping columns (that will change attnums) */ ALTER TABLE ints DROP COLUMN c1; ALTER TABLE ints ADD COLUMN c1 SMALLINT; INSERT INTO ints (c1, c2, c3, c4) SELECT i, i + 1, i + 2, i+ 3 FROM generate_series(7, 8) i; SELECT c1, c2, c3, c4 FROM ints ORDER BY c1; c1 | c2 | c3 | c4 ----+----+----+---- 1 | 2 | 3 | 4 2 | 3 | 4 | 5 3 | 4 | 5 | 6 4 | 7 | 6 | 5 5 | 8 | 7 | 6 6 | 9 | 8 | 7 7 | 8 | 9 | 10 8 | 9 | 10 | 11 (8 rows) /* check other number types */ INSERT INTO uints SELECT i, i + 1, i + 2, i+ 3, (i % 2)::bool FROM generate_series(1, 3) i; SELECT * FROM uints ORDER BY c1; c1 | c2 | c3 | c4 | c5 ----+----+----+----+---- 1 | 2 | 3 | 4 | t 2 | 3 | 4 | 5 | f 3 | 4 | 5 | 6 | t (3 rows) INSERT INTO floats SELECT i * 1.1, i + 2.1 FROM generate_series(1, 3) i; SELECT * FROM floats ORDER BY c1; c1 | c2 -----+----- 1.1 | 3.1 2.2 | 4.1 3.3 | 5.1 (3 rows) /* check nullable */ INSERT INTO null_ints SELECT i, case WHEN i % 2 = 0 THEN NULL ELSE i END FROM generate_series(1, 10) i; INSERT INTO null_ints(c1) SELECT i FROM generate_series(11, 13) i; SELECT * FROM null_ints ORDER BY c1; c1 | c2 ----+---- 1 | 1 2 | 3 | 3 4 | 5 | 5 6 | 7 | 7 8 | 9 | 9 10 | 11 | 12 | 13 | (13 rows) SELECT * FROM null_ints ORDER BY c1; c1 | c2 ----+---- 1 | 1 2 | 3 | 3 4 | 5 | 5 6 | 7 | 7 8 | 9 | 9 10 | 11 | 12 | 13 | (13 rows) /* check dates and strings */ ALTER TABLE complex ALTER COLUMN c8 SET DATA TYPE timestamp(3); \d+ complex Foreign table "binary_inserts_test.complex" Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description --------+--------------------------------+-----------+----------+---------+-------------+----------+--------------+------------- c1 | integer | | not null | | | plain | | c2 | date | | not null | | | plain | | c3 | timestamp with time zone | | not null | | | plain | | c4 | text | | not null | | | extended | | c5 | character varying(10) | | not null | | | extended | | c6 | text | | not null | | | extended | | c7 | date | | not null | | | plain | | c8 | timestamp(3) without time zone | | not null | | | plain | | Not-null constraints: "complex_c1_not_null" NOT NULL "c1" "complex_c2_not_null" NOT NULL "c2" "complex_c3_not_null" NOT NULL "c3" "complex_c4_not_null" NOT NULL "c4" "complex_c5_not_null" NOT NULL "c5" "complex_c6_not_null" NOT NULL "c6" "complex_c7_not_null" NOT NULL "c7" "complex_c8_not_null" NOT NULL "c8" Server: binary_inserts_loopback FDW options: (database 'binary_inserts_test', table_name 'complex', engine 'MergeTree') INSERT INTO complex VALUES (1, '2020-06-01', '2020-06-02 10:01:02', 't1', 'fix_t1', 'low1', '2020-06-01', '2020-06-02 10:01:02.123'), (2, '2020-06-02', '2020-06-03 10:01:02', 5, 'fix_t2', 'low2', '2020-06-02', '2020-06-03 11:01:02.234'), (3, '2020-06-03', '2020-06-04 10:01:02', 5, 'fix_t3', 'low3', '2020-06-03', '2020-06-04 12:01:02'); SET session timezone = 'UTC'; INSERT INTO complex VALUES (4, '1970-01-01', '1970-01-01 00:00:00', 5, 'fix_t4', 'low4', '1970-01-01', '1970-01-01 00:00:00'), (5, '2000-01-01', '2000-01-01 00:00:00', 5, 'fix_t5', 'low5', '2000-01-01', '2000-01-01 00:00:00'); SELECT * FROM complex ORDER BY c1; c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 ----+------------+------------------------+----+--------+------+------------+------------------------- 1 | 2020-06-01 | 2020-06-02 17:01:02+00 | t1 | fix_t1 | low1 | 2020-06-01 | 2020-06-02 17:01:02.123 2 | 2020-06-02 | 2020-06-03 17:01:02+00 | 5 | fix_t2 | low2 | 2020-06-02 | 2020-06-03 18:01:02.234 3 | 2020-06-03 | 2020-06-04 17:01:02+00 | 5 | fix_t3 | low3 | 2020-06-03 | 2020-06-04 19:01:02 4 | 1970-01-01 | 1970-01-01 00:00:00+00 | 5 | fix_t4 | low4 | 1970-01-01 | 1970-01-01 00:00:00 5 | 2000-01-01 | 2000-01-01 00:00:00+00 | 5 | fix_t5 | low5 | 2000-01-01 | 2000-01-01 00:00:00 (5 rows) RESET timezone; SELECT * FROM complex ORDER BY c1; c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 ----+------------+------------------------+----+--------+------+------------+------------------------- 1 | 2020-06-01 | 2020-06-02 10:01:02-07 | t1 | fix_t1 | low1 | 2020-06-01 | 2020-06-02 10:01:02.123 2 | 2020-06-02 | 2020-06-03 10:01:02-07 | 5 | fix_t2 | low2 | 2020-06-02 | 2020-06-03 11:01:02.234 3 | 2020-06-03 | 2020-06-04 10:01:02-07 | 5 | fix_t3 | low3 | 2020-06-03 | 2020-06-04 12:01:02 4 | 1970-01-01 | 1969-12-31 16:00:00-08 | 5 | fix_t4 | low4 | 1970-01-01 | 1969-12-31 16:00:00 5 | 2000-01-01 | 1999-12-31 16:00:00-08 | 5 | fix_t5 | low5 | 2000-01-01 | 1999-12-31 16:00:00 (5 rows) /* check arrays */ INSERT INTO arrays VALUES (1, ARRAY[1,2], ARRAY['x']), (2, ARRAY[3,4,5], ARRAY['😀', '🦁']), (3, ARRAY[6,4], ARRAY['Zippy', 'Lisa "Skippy" Taylor']), (4, '{}'::int[], '{"[bracket''d]"}'::text[]), (5, ARRAY[0], ARRAY['[]']), (5, ARRAY[0], ARRAY[E'\\\b\f\r\n\t\a\'']); SELECT * FROM arrays ORDER BY c1; c1 | c2 | c3 ----+---------+---------------------------------- 1 | {1,2} | {x} 2 | {3,4,5} | {😀,🦁} 3 | {6,4} | {Zippy,"Lisa \"Skippy\" Taylor"} 4 | {} | {[bracket'd]} 5 | {0} | {[]} 5 | {0} | {"\\\x08\x0C\r + | | a'"} (6 rows) /* Check UUIDs and IPs */ \d addr Foreign table "binary_inserts_test.addr" Column | Type | Collation | Nullable | Default | FDW options --------+------+-----------+----------+---------+------------- c1 | uuid | | not null | | c2 | inet | | not null | | c3 | inet | | not null | | Server: binary_inserts_loopback FDW options: (database 'binary_inserts_test', table_name 'addr', engine 'MergeTree') INSERT INTO addr VALUES ('61f0c404-5cb3-11e7-907b-a6006ad3dba0', '116.106.34.242', '2001:44c8:129:2632:33:0:252:2'), ('00000000-0000-0000-0000-000000000000', '127.0.0.1', '::ffff:127.0.0.1'), ('C62848ED-7316-4D15-92F3-9BB71EB69640', '183.247.232.58', '2a02:e980:1e::1') ; SELECT * FROM addr ORDER BY c1; c1 | c2 | c3 --------------------------------------+----------------+------------------------------- 00000000-0000-0000-0000-000000000000 | 127.0.0.1 | ::ffff:127.0.0.1 61f0c404-5cb3-11e7-907b-a6006ad3dba0 | 116.106.34.242 | 2001:44c8:129:2632:33:0:252:2 c62848ed-7316-4d15-92f3-9bb71eb69640 | 183.247.232.58 | 2a02:e980:1e::1 (3 rows) /* Check BYTEA */ CREATE FOREIGN TABLE bbytes( c1 int, c2 BYTEA, c3 BYTEA ) SERVER binary_inserts_loopback OPTIONS (table_name 'bytes'); INSERT INTO bbytes SELECT n, sha224(bytea('val'||n)), decode(md5('int'||n), 'hex') FROM generate_series(1, 4) n; -- Should have full binary values, including nul bytes, from BYTEA columns. SELECT * FROM bbytes ORDER BY c1; c1 | c2 | c3 ----+------------------------------------------------------------+------------------------------------ 1 | \x1bf7f0cc821d31178616a55a8e0c52677735397cdde6f4153a9fd3d7 | \xae3b28cde02542f81acce8783245430d 2 | \x5f6e9e12cd8592712e638016f4b1a2e73230ee40db498c0f0b1dc841 | \x23e7c6cacb8383f878ad093b0027d72b 3 | \x53ac2c1fa83c8f64603fe9568d883331007d6281de330a4b5e728f9e | \x7e969132fc656148b97b6a2ee8bc83c1 4 | \x4e3c2e4cb7542a45173a8dac939ddc4bc75202e342ebc769b0f5da2f | \x8ef30f44c65480d12b650ab6b2b04245 (4 rows) -- Nul bytes should truncate TEXT columns. SELECT c1, encode(c2::bytea, 'hex'), encode(c3::bytea, 'hex') FROM bytes ORDER BY c1; c1 | encode | encode ----+----------------------------------------------------------+---------------------------------- 1 | 1bf7f0cc821d31178616a55a8e0c52677735397cdde6f4153a9fd3d7 | ae3b28cde02542f81acce8783245430d 2 | 5f6e9e12cd8592712e638016f4b1a2e73230ee40db498c0f0b1dc841 | 23e7c6cacb8383f878ad093b 3 | 53ac2c1fa83c8f64603fe9568d883331 | 7e969132fc656148b97b6a2ee8bc83c1 4 | 4e3c2e4cb7542a45173a8dac939ddc4bc75202e342ebc769b0f5da2f | 8ef30f44c65480d12b650ab6b2b04245 (4 rows) SELECT clickhouse_raw_query('TRUNCATE binary_inserts_test.bytes'); clickhouse_raw_query ---------------------- (1 row) -- Should fail. INSERT INTO bytes SELECT n, sha224(bytea('val'||n)), decode(md5('int'||n), 'hex') FROM generate_series(1, 4) n; ERROR: value too long for type character varying(16) -- Remove FixedString length. ALTER FOREIGN TABLE bytes ALTER c3 TYPE TEXT; SELECT clickhouse_raw_query('ALTER TABLE binary_inserts_test.bytes MODIFY COLUMN c3 String'); clickhouse_raw_query ---------------------- (1 row) -- Should succeed. INSERT INTO bytes SELECT n, sha224(bytea('val'||n)), decode(md5('int'||n), 'hex') FROM generate_series(1, 4) n; SELECT * FROM bbytes ORDER BY c1; c1 | c2 | c3 ----+------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------ 1 | \x5c783162663766306363383231643331313738363136613535613865306335323637373733353339376364646536663431353361396664336437 | \x5c786165336232386364653032353432663831616363653837383332343534333064 2 | \x5c783566366539653132636438353932373132653633383031366634623161326537333233306565343064623439386330663062316463383431 | \x5c783233653763366361636238333833663837386164303933623030323764373262 3 | \x5c783533616332633166613833633866363436303366653935363864383833333331303037643632383164653333306134623565373238663965 | \x5c783765393639313332666336353631343862393762366132656538626338336331 4 | \x5c783465336332653463623735343261343531373361386461633933396464633462633735323032653334326562633736396230663564613266 | \x5c783865663330663434633635343830643132623635306162366232623034323435 (4 rows) SELECT c1, encode(c2::bytea, 'hex'), encode(c3::bytea, 'hex') FROM bytes ORDER BY c1; c1 | encode | encode ----+----------------------------------------------------------+---------------------------------- 1 | 1bf7f0cc821d31178616a55a8e0c52677735397cdde6f4153a9fd3d7 | ae3b28cde02542f81acce8783245430d 2 | 5f6e9e12cd8592712e638016f4b1a2e73230ee40db498c0f0b1dc841 | 23e7c6cacb8383f878ad093b0027d72b 3 | 53ac2c1fa83c8f64603fe9568d883331007d6281de330a4b5e728f9e | 7e969132fc656148b97b6a2ee8bc83c1 4 | 4e3c2e4cb7542a45173a8dac939ddc4bc75202e342ebc769b0f5da2f | 8ef30f44c65480d12b650ab6b2b04245 (4 rows) -- Test NULL values. SELECT clickhouse_raw_query($$ CREATE TABLE binary_inserts_test.null_vals ( c1 UInt8, -- INT2OID c2 Nullable(UInt8), c3 Nullable(Int8), c4 Nullable(Int16), -- INT4OID c5 Nullable(Int32), c6 Nullable(UInt16), -- INT8OID c7 Nullable(Int64), c8 Nullable(UInt32), c9 Nullable(UInt64), -- FLOAT4OID, FLOAT8OID c10 Nullable(Float32), c11 Nullable(Float64), -- NUMERICOID c12 Nullable(Decimal128(1)), c13 Nullable(Decimal64(1)), c14 Nullable(Decimal32(1)), c15 Nullable(Decimal(1, 0)), -- TEXTOID c16 Nullable(String), c17 Nullable(FixedString(1)), c18 Nullable(Enum('x'=1)), c19 Nullable(Enum16('x'=1)), -- c20 LowCardinality(Nullable(String)), -- DATEOID c21 Nullable(Date), c22 Nullable(Date32), -- TIMESTAMPOID, TIMESTAMPTZOID c23 Nullable(DateTime), c24 Nullable(DateTime64), -- ANYARRAYOID -- c25 Array(Nullable(Int32)), -- pg_clickhouse: nested Nullable is not supported -- UUIDOID c26 Nullable(UUID), -- INETOID c27 Nullable(IPv4), c28 Nullable(IPv6) ) ENGINE = MergeTree ORDER BY (c1) $$); clickhouse_raw_query ---------------------- (1 row) IMPORT FOREIGN SCHEMA binary_inserts_test LIMIT TO (null_vals) FROM SERVER binary_inserts_loopback INTO binary_inserts_test; INSERT INTO null_vals VALUES( 1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, -- NULL, NULL, NULL, NULL, NULL, -- ARRAY[NULL]::int[], NULL, NULL, NULL ); SELECT * FROM null_vals; c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 | c11 | c12 | c13 | c14 | c15 | c16 | c17 | c18 | c19 | c21 | c22 | c23 | c24 | c26 | c27 | c28 ----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+----- 1 | | | | | | | | | | | | | | | | | | | | | | | | | (1 row) -- Test default values. SELECT clickhouse_raw_query($$ CREATE TABLE binary_inserts_test.default_vals ( c1 UInt8, -- INT2OID c2 UInt8, c3 Int8, c4 Int16, -- INT4OID c5 Int32, c6 UInt16, -- INT8OID c7 Int64, c8 UInt32, c9 UInt64, -- FLOAT4OID, FLOAT8OID c10 Float32, c11 Float64, -- NUMERICOID c12 Decimal128(1), c13 Decimal64(1), c14 Decimal32(1), c15 Decimal(1, 0), -- TEXTOID c16 String, c17 FixedString(1), c18 Enum('x'=1), c19 Enum16('x'=1), c20 LowCardinality(String), -- DATEOID c21 Date, c22 Date32, -- TIMESTAMPOID, TIMESTAMPTZOID c23 DateTime, c24 DateTime64, -- ANYARRAYOID c25 Array(Int32), -- pg_clickhouse: nested Nullable is not supported -- UUIDOID c26 UUID, -- INETOID c27 IPv4, c28 IPv6 ) ENGINE = MergeTree ORDER BY (c1) $$); clickhouse_raw_query ---------------------- (1 row) IMPORT FOREIGN SCHEMA binary_inserts_test LIMIT TO (default_vals) FROM SERVER binary_inserts_loopback INTO binary_inserts_test; -- Fails on c2, the first column we try to set to NULL. Will cease to fail if -- the `if (isnull && !nullable)` block is removed from column_append(). See -- its comment for details. INSERT INTO default_vals VALUES( 1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, ARRAY[NULL]::int[], NULL, NULL, NULL ); ERROR: pg_clickhouse: could not append data to column - cannot append NULL to NOT NULL UInt8 column SELECT * FROM default_vals; c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 | c11 | c12 | c13 | c14 | c15 | c16 | c17 | c18 | c19 | c20 | c21 | c22 | c23 | c24 | c25 | c26 | c27 | c28 ----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+----- (0 rows) -- Test unsupported Nullables. INSERT INTO not_nullable VALUES (1, 'x', 'x', NULL); ERROR: pg_clickhouse: could not append data to column - cannot append NULL to NOT NULL LowCardinality(Nullable(String)) column DROP USER MAPPING FOR CURRENT_USER SERVER binary_inserts_loopback; SELECT clickhouse_raw_query('DROP DATABASE binary_inserts_test'); clickhouse_raw_query ---------------------- (1 row) DROP SERVER binary_inserts_loopback CASCADE; NOTICE: drop cascades to 12 other objects DETAIL: drop cascades to foreign table addr drop cascades to foreign table arrays drop cascades to foreign table bytes drop cascades to foreign table complex drop cascades to foreign table floats drop cascades to foreign table ints drop cascades to foreign table not_nullable drop cascades to foreign table null_ints drop cascades to foreign table uints drop cascades to foreign table bbytes drop cascades to foreign table null_vals drop cascades to foreign table default_vals