-- Partitioned queues: msg_id becomes GENERATED BY DEFAULT AS IDENTITY. -- pg_partman's partition_data_* tooling moves rows out of the default -- partition by inserting them with their existing msg_id, which -- GENERATED ALWAYS refuses. pgmq's own writers never supply msg_id, so this -- changes nothing for them. DO $$ DECLARE queue_record RECORD; qtable TEXT; BEGIN FOR queue_record IN SELECT queue_name FROM pgmq.meta WHERE is_partitioned LOOP qtable := pgmq.format_table_name(queue_record.queue_name, 'q'); IF EXISTS ( SELECT 1 FROM information_schema.columns WHERE table_schema = 'pgmq' AND table_name = qtable AND column_name = 'msg_id' AND identity_generation = 'ALWAYS' ) THEN EXECUTE FORMAT('ALTER TABLE pgmq.%I ALTER COLUMN msg_id SET GENERATED BY DEFAULT', qtable); END IF; END LOOP; END; $$; -- create_partitioned gains a premake parameter. The three-argument signature is -- dropped rather than kept alongside it, so existing calls resolve to one function. DROP FUNCTION IF EXISTS pgmq.create_partitioned(TEXT, TEXT, TEXT); CREATE FUNCTION pgmq.create_partitioned( queue_name TEXT, partition_interval TEXT DEFAULT '10000', retention_interval TEXT DEFAULT '100000', premake INTEGER DEFAULT 4 ) RETURNS void AS $$ DECLARE partition_col TEXT; a_partition_col TEXT; qtable TEXT := pgmq.format_table_name(queue_name, 'q'); qtable_seq TEXT := qtable || '_msg_id_seq'; atable TEXT := pgmq.format_table_name(queue_name, 'a'); fq_qtable TEXT := 'pgmq.' || qtable; fq_atable TEXT := 'pgmq.' || atable; BEGIN PERFORM pgmq.validate_queue_name(queue_name); PERFORM pgmq.acquire_queue_lock(queue_name); PERFORM pgmq._ensure_pg_partman_installed(); IF premake < 1 THEN RAISE EXCEPTION 'premake must be at least 1, got %', premake; END IF; SELECT pgmq._get_partition_col(partition_interval) INTO partition_col; EXECUTE FORMAT( $QUERY$ CREATE TABLE IF NOT EXISTS pgmq.%I ( msg_id BIGINT GENERATED BY DEFAULT AS IDENTITY, read_ct INT DEFAULT 0 NOT NULL, enqueued_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL, last_read_at TIMESTAMP WITH TIME ZONE, vt TIMESTAMP WITH TIME ZONE NOT NULL, message JSONB, headers JSONB ) PARTITION BY RANGE (%I) $QUERY$, qtable, partition_col ); -- https://github.com/pgpartman/pg_partman/blob/master/doc/pg_partman.md -- p_parent_table - the existing parent table. MUST be schema qualified, even if in public schema. EXECUTE FORMAT( $QUERY$ SELECT %I.create_parent( p_parent_table := %L, p_control := %L, p_interval := %L, p_premake := %s, p_type := case when pgmq._get_pg_partman_major_version() = 5 then 'range' else 'native' end ) $QUERY$, pgmq._get_pg_partman_schema(), fq_qtable, partition_col, partition_interval, premake ); EXECUTE FORMAT( $QUERY$ CREATE INDEX IF NOT EXISTS %I ON pgmq.%I (%I); $QUERY$, qtable || '_part_idx', qtable, partition_col ); EXECUTE FORMAT( $QUERY$ UPDATE %I.part_config SET retention = %L, retention_keep_table = false, retention_keep_index = true, automatic_maintenance = 'on' WHERE parent_table = %L; $QUERY$, pgmq._get_pg_partman_schema(), retention_interval, 'pgmq.' || qtable ); EXECUTE FORMAT( $QUERY$ INSERT INTO pgmq.meta (queue_name, is_partitioned, is_unlogged) VALUES (%L, true, false) ON CONFLICT DO NOTHING; $QUERY$, queue_name ); IF partition_col = 'enqueued_at' THEN a_partition_col := 'archived_at'; ELSE a_partition_col := partition_col; END IF; EXECUTE FORMAT( $QUERY$ CREATE TABLE IF NOT EXISTS pgmq.%I ( msg_id BIGINT NOT NULL, read_ct INT DEFAULT 0 NOT NULL, enqueued_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL, last_read_at TIMESTAMP WITH TIME ZONE, archived_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL, vt TIMESTAMP WITH TIME ZONE NOT NULL, message JSONB, headers JSONB ) PARTITION BY RANGE (%I); $QUERY$, atable, a_partition_col ); -- https://github.com/pgpartman/pg_partman/blob/master/doc/pg_partman.md -- p_parent_table - the existing parent table. MUST be schema qualified, even if in public schema. EXECUTE FORMAT( $QUERY$ SELECT %I.create_parent( p_parent_table := %L, p_control := %L, p_interval := %L, p_premake := %s, p_type := case when pgmq._get_pg_partman_major_version() = 5 then 'range' else 'native' end ) $QUERY$, pgmq._get_pg_partman_schema(), fq_atable, a_partition_col, partition_interval, premake ); EXECUTE FORMAT( $QUERY$ UPDATE %I.part_config SET retention = %L, retention_keep_table = false, retention_keep_index = true, automatic_maintenance = 'on' WHERE parent_table = %L; $QUERY$, pgmq._get_pg_partman_schema(), retention_interval, 'pgmq.' || atable ); EXECUTE FORMAT( $QUERY$ CREATE INDEX IF NOT EXISTS %I ON pgmq.%I (archived_at); $QUERY$, 'archived_at_idx_' || queue_name, atable ); END; $$ LANGUAGE plpgsql; -- metrics_result gains default_partition_length, which metrics() now reports. ALTER TYPE pgmq.metrics_result ADD ATTRIBUTE default_partition_length bigint; -- get metrics for a single queue CREATE OR REPLACE FUNCTION pgmq.metrics(queue_name TEXT) RETURNS pgmq.metrics_result AS $$ DECLARE result_row pgmq.metrics_result; query TEXT; qtable TEXT := pgmq.format_table_name(queue_name, 'q'); q_default_partition TEXT := qtable || '_default'; a_default_partition TEXT := pgmq.format_table_name(queue_name, 'a') || '_default'; default_partition_length BIGINT; BEGIN -- Only partitioned queues have default partitions. Messages in them have no -- partition of their own, which means pg_partman maintenance is failing for -- this queue; a non-zero value here is the signal to act on. The planner's -- estimate is used so that a large spill does not slow down every scrape. IF to_regclass(FORMAT('pgmq.%I', q_default_partition)) IS NOT NULL THEN SELECT COALESCE(SUM(GREATEST(c.reltuples, 0))::bigint, 0) INTO default_partition_length FROM pg_class c WHERE c.oid IN ( to_regclass(FORMAT('pgmq.%I', q_default_partition)), to_regclass(FORMAT('pgmq.%I', a_default_partition)) ); END IF; query := FORMAT( $QUERY$ WITH q_summary AS ( SELECT count(*) as queue_length, count(CASE WHEN vt <= NOW() THEN 1 END) as queue_visible_length, EXTRACT(epoch FROM (NOW() - max(enqueued_at)))::int as newest_msg_age_sec, EXTRACT(epoch FROM (NOW() - min(enqueued_at)))::int as oldest_msg_age_sec, NOW() as scrape_time FROM pgmq.%I ), all_metrics AS ( SELECT CASE WHEN is_called THEN last_value ELSE 0 END as total_messages FROM pgmq.%I ) SELECT %L as queue_name, q_summary.queue_length, q_summary.newest_msg_age_sec, q_summary.oldest_msg_age_sec, all_metrics.total_messages, q_summary.scrape_time, q_summary.queue_visible_length, %L::bigint as default_partition_length FROM q_summary, all_metrics $QUERY$, qtable, qtable || '_msg_id_seq', queue_name, default_partition_length ); EXECUTE query INTO result_row; RETURN result_row; END; $$ LANGUAGE plpgsql;