-- -- Failure tests for creating reference table -- CREATE SCHEMA failure_reference_table; SET search_path TO 'failure_reference_table'; SET citus.next_shard_id TO 10000000; SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- (1 row) -- this is merely used to get the schema creation propagated. Without there are failures -- not related to reference tables but schema creation due to dependency creation on workers CREATE TYPE schema_proc AS (a int); DROP TYPE schema_proc; CREATE TABLE ref_table(id int); INSERT INTO ref_table VALUES(1),(2),(3); -- Kill on sending first query to worker node, should error -- out and not create any placement SELECT citus.mitmproxy('conn.onQuery().kill()'); mitmproxy --------------------------------------------------------------------- (1 row) SELECT create_reference_table('ref_table'); WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx ERROR: failure on connection marked as essential: localhost:xxxxx SELECT count(*) FROM pg_dist_shard_placement; count --------------------------------------------------------------------- 0 (1 row) -- Kill after creating transaction on worker node SELECT citus.mitmproxy('conn.onCommandComplete(command="BEGIN").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) SELECT create_reference_table('ref_table'); WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx ERROR: failure on connection marked as essential: localhost:xxxxx SELECT count(*) FROM pg_dist_shard_placement; count --------------------------------------------------------------------- 0 (1 row) -- Cancel after creating transaction on worker node SELECT citus.mitmproxy('conn.onCommandComplete(command="BEGIN").cancel(' || pg_backend_pid() || ')'); mitmproxy --------------------------------------------------------------------- (1 row) SELECT create_reference_table('ref_table'); ERROR: canceling statement due to user request SELECT count(*) FROM pg_dist_shard_placement; count --------------------------------------------------------------------- 0 (1 row) -- Kill after copying data to worker node SELECT citus.mitmproxy('conn.onCommandComplete(command="SELECT 1").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) SELECT create_reference_table('ref_table'); WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx ERROR: failure on connection marked as essential: localhost:xxxxx SELECT count(*) FROM pg_dist_shard_placement; count --------------------------------------------------------------------- 0 (1 row) -- Cancel after copying data to worker node SELECT citus.mitmproxy('conn.onCommandComplete(command="SELECT 1").cancel(' || pg_backend_pid() || ')'); mitmproxy --------------------------------------------------------------------- (1 row) SELECT create_reference_table('ref_table'); ERROR: canceling statement due to user request SELECT count(*) FROM pg_dist_shard_placement; count --------------------------------------------------------------------- 0 (1 row) -- Kill after copying data to worker node SELECT citus.mitmproxy('conn.onCommandComplete(command="COPY 3").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) SELECT create_reference_table('ref_table'); NOTICE: Copying data from local table... ERROR: failed to COPY to shard xxxxx on localhost:xxxxx SELECT count(*) FROM pg_dist_shard_placement; count --------------------------------------------------------------------- 0 (1 row) -- Cancel after copying data to worker node SELECT citus.mitmproxy('conn.onCommandComplete(command="COPY 3").cancel(' || pg_backend_pid() || ')'); mitmproxy --------------------------------------------------------------------- (1 row) SELECT create_reference_table('ref_table'); NOTICE: Copying data from local table... ERROR: canceling statement due to user request SELECT count(*) FROM pg_dist_shard_placement; count --------------------------------------------------------------------- 0 (1 row) -- we don't want to see the prepared transaction numbers in the warnings SET client_min_messages TO ERROR; -- Kill after preparing transaction. Since we don't commit after preparing, we recover -- prepared transaction afterwards. SELECT citus.mitmproxy('conn.onCommandComplete(command="PREPARE TRANSACTION").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) SELECT create_reference_table('ref_table'); ERROR: connection not open CONTEXT: while executing command on localhost:xxxxx SELECT count(*) FROM pg_dist_shard_placement; count --------------------------------------------------------------------- 0 (1 row) SELECT recover_prepared_transactions(); recover_prepared_transactions --------------------------------------------------------------------- 1 (1 row) -- Kill after commiting prepared, this should succeed SELECT citus.mitmproxy('conn.onCommandComplete(command="COMMIT PREPARED").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) SELECT create_reference_table('ref_table'); create_reference_table --------------------------------------------------------------------- (1 row) SELECT shardid, nodeport, shardstate FROM pg_dist_shard_placement ORDER BY shardid, nodeport; shardid | nodeport | shardstate --------------------------------------------------------------------- 10000003 | 9060 | 1 10000003 | 57637 | 1 (2 rows) SET client_min_messages TO NOTICE; SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- (1 row) DROP TABLE ref_table; DROP SCHEMA failure_reference_table; SET citus.enable_ddl_propagation TO OFF; CREATE SCHEMA failure_reference_table; SET citus.enable_ddl_propagation TO ON; CREATE TABLE ref_table(id int); INSERT INTO ref_table VALUES(1),(2),(3); -- Test in transaction SELECT citus.mitmproxy('conn.onQuery().kill()'); mitmproxy --------------------------------------------------------------------- (1 row) BEGIN; SELECT create_reference_table('ref_table'); WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx ERROR: failure on connection marked as essential: localhost:xxxxx COMMIT; -- kill on ROLLBACK, should be rollbacked SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) BEGIN; SELECT create_reference_table('ref_table'); NOTICE: Copying data from local table... NOTICE: copying the data has completed DETAIL: The local data in the table is no longer visible, but is still on disk. HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$failure_reference_table.ref_table$$) create_reference_table --------------------------------------------------------------------- (1 row) ROLLBACK; WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx SELECT * FROM pg_dist_shard_placement ORDER BY shardid, nodeport; shardid | shardstate | shardlength | nodename | nodeport | placementid --------------------------------------------------------------------- (0 rows) -- cancel when the coordinator send ROLLBACK, should be rollbacked. We ignore cancellations -- during the ROLLBACK. SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").cancel(' || pg_backend_pid() || ')'); mitmproxy --------------------------------------------------------------------- (1 row) BEGIN; SELECT create_reference_table('ref_table'); NOTICE: Copying data from local table... NOTICE: copying the data has completed DETAIL: The local data in the table is no longer visible, but is still on disk. HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$failure_reference_table.ref_table$$) create_reference_table --------------------------------------------------------------------- (1 row) ROLLBACK; SELECT * FROM pg_dist_shard_placement ORDER BY shardid, nodeport; shardid | shardstate | shardlength | nodename | nodeport | placementid --------------------------------------------------------------------- (0 rows) DROP SCHEMA failure_reference_table CASCADE; NOTICE: drop cascades to table ref_table SET search_path TO default;