-- Damaged items: an ERROR, never a PANIC, a crash or a silent overwrite. -- -- 1. A container item whose line pointer claims more than its slack: every -- reader and writer refuses it (lion_page_item_fetch()). The push-down -- of the root used to copy it by that length inside its record, which in -- rmgr mode (installcheck-rmgr runs this file too) is a critical section: -- a PANIC, again after every recovery. -- 2. One whose line pointer reaches into the next item but stays within its -- slack: readers are right to take it, but the in-place insert grew the -- container into its neighbour and logged it (lion_page_check_alone()). -- 3. A stored text key whose header says TOAST pointer: the first descent -- that compared against it crashed in detoast (lion_check_key()). -- 4. A member at heap offset 0, which lion_index_verify() passed. -- 5. A NARROW (DESIGN.md §38) whose cardinality is not its payload's. -- 6. A NARROW of a width the build does not have, which sizes it. -- 7. A directory entry whose key length claims more than its item holds. -- 8. An entry whose TID count is less than what VACUUM removes from it. -- 9. An INLINE entry whose item ends between its key and the end of its -- positions extension, where the payload length wrapped around zero. -- 10. An index whose meta page says another format version than this build's. -- 11. A directory entry whose line pointer reaches into a CHAIN entry, which -- every record of an insert into that entry's posting set rewrites in -- place, by its own line pointer (lion_page_check_alone()). -- 12. Two chunks of a position tree's leaf at the same bytes, which VACUUM -- wrote back by their line pointers, one over the other -- (lion_page_check_items()). -- -- The pages are damaged on disk, as corrupt.sql does it (pg_buffercache_evict(), -- PostgreSQL 17 and later - on 16 the test is skipped, -- test/expected/corrupt_items_1.out). The byte offsets assume a -- little-endian server. SET client_min_messages = warning; CREATE EXTENSION IF NOT EXISTS pg_lion; CREATE EXTENSION IF NOT EXISTS pageinspect; CREATE EXTENSION IF NOT EXISTS pg_buffercache; RESET client_min_messages; SELECT current_setting('server_version_num')::int >= 170000 AS lion_di_can_evict \gset \if :lion_di_can_evict SET statement_timeout = '60s'; -- A little-endian uint32 at byte off of a bytea, and one as four bytes. CREATE FUNCTION lion_di_u32(f bytea, off int) RETURNS bigint LANGUAGE sql IMMUTABLE AS $$ SELECT get_byte(f, off)::bigint | (get_byte(f, off + 1)::bigint << 8) | (get_byte(f, off + 2)::bigint << 16) | (get_byte(f, off + 3)::bigint << 24) $$; CREATE FUNCTION lion_di_le32(v bigint) RETURNS bytea LANGUAGE sql IMMUTABLE AS $$ SELECT set_byte(set_byte(set_byte(set_byte('\x00000000'::bytea, 0, (v & 255)::int), 1, ((v >> 8) & 255)::int), 2, ((v >> 16) & 255)::int), 3, ((v >> 24) & 255)::int) $$; -- Block blk of an index's main fork, read from the file. CREATE FUNCTION lion_di_block(idx regclass, blk bigint) RETURNS bytea LANGUAGE sql AS $$ SELECT pg_read_binary_file(pg_relation_filepath(idx), blk * current_setting('block_size')::int, current_setting('block_size')::int) $$; -- Line pointer off of a page: lp_off is its low 15 bits, lp_len its top 15. CREATE FUNCTION lion_di_item(page bytea, off int) RETURNS int LANGUAGE sql IMMUTABLE AS $$ SELECT (lion_di_u32(page, 24 + 4 * (off - 1)) & 32767)::int $$; CREATE FUNCTION lion_di_lplen(page bytea, off int) RETURNS int LANGUAGE sql IMMUTABLE AS $$ SELECT (lion_di_u32(page, 24 + 4 * (off - 1)) >> 17)::int $$; -- Write an index out and drop its pages from the buffer pool. CREATE FUNCTION lion_di_evict(idx regclass) RETURNS void LANGUAGE plpgsql AS $$ BEGIN CHECKPOINT; PERFORM pg_buffercache_evict(bufferid) FROM pg_buffercache WHERE relfilenode = pg_relation_filenode(idx) AND reldatabase = (SELECT oid FROM pg_database WHERE datname = current_database()); END $$; -- Overwrite bytes at byte pos of block blk, with the page's new checksum. CREATE FUNCTION lion_di_poke(idx regclass, blk bigint, pos int, bytes bytea) RETURNS void LANGUAGE plpgsql AS $$ DECLARE path text := pg_relation_filepath(idx); bs int := current_setting('block_size')::int; f bytea := pg_read_binary_file(path); page bytea; ck int; lo oid; BEGIN page := substring(f FROM (blk * bs)::int + 1 FOR bs); page := overlay(page PLACING bytes FROM pos + 1); ck := page_checksum(page, blk::int)::int & 65535; page := overlay(page PLACING set_byte(set_byte('\x0000'::bytea, 0, ck & 255), 1, ck >> 8) FROM 9); f := overlay(f PLACING page FROM (blk * bs)::int + 1); lo := lo_from_bytea(0, f); PERFORM lo_export(lo, path); PERFORM lo_unlink(lo); END $$; -- Give line pointer off of block blk the length len, keeping its offset. CREATE FUNCTION lion_di_set_lplen(idx regclass, blk bigint, off int, len int) RETURNS void LANGUAGE sql AS $$ SELECT lion_di_poke(idx, blk, 24 + 4 * (off - 1), lion_di_le32((lion_di_u32(lion_di_block(idx, blk), 24 + 4 * (off - 1)) & 131071) | (len::bigint << 17))) $$; -- Run a statement; report its SQLSTATE and its message with the numbers out. CREATE FUNCTION lion_di_try(q text) RETURNS text LANGUAGE plpgsql AS $$ BEGIN EXECUTE q; RETURN 'ok'; EXCEPTION WHEN OTHERS THEN RETURN SQLSTATE || ': ' || regexp_replace(SQLERRM, '[0-9]+', 'N', 'g'); END $$; SET enable_seqscan = off; -- ---------- 1. a line pointer past its item's slack, and the push-down ---------- -- Key 7 is every 16th row of 256 heap blocks: its posting set is one leaf, the -- root, holding four ARRAY containers of 1816 bytes. The fourth one's line -- pointer claims 4000. Growing the set by a fifth container then pushes the -- root down, which moved the items by their line pointers. CREATE TABLE lion_di (k int NOT NULL); INSERT INTO lion_di SELECT CASE WHEN i % 16 = 0 THEN 7 ELSE 1 END FROM generate_series(0, 4 * 14464 - 1) i; CREATE INDEX lion_di_i ON lion_di USING lion (k); SELECT lion_di_evict('lion_di_i'); lion_di_evict --------------- (1 row) SELECT lion_index_posting_root('lion_di_i', 7) AS lion_di_root \gset SELECT lion_di_lplen(lion_di_block('lion_di_i', :lion_di_root), 4) AS lp_len; lp_len -------- 1816 (1 row) SELECT lion_di_set_lplen('lion_di_i', :lion_di_root, 4, 4000); lion_di_set_lplen ------------------- (1 row) SELECT lion_di_try('SELECT count(*) FROM lion_di WHERE k = 7'); lion_di_try ------------------------------------------------------------------------------------------------------------------------------------- XX002: lion index "lion_di_i": item N on container page N occupies N bytes, N more than its N bytes need (at most N bytes of slack) (1 row) SELECT lion_di_try($$INSERT INTO lion_di SELECT CASE WHEN i % 2 = 0 THEN 7 ELSE 1 END FROM generate_series(1, 20000) i$$); lion_di_try ------------------------------------------------------------------------------------------------------------------------------------- XX002: lion index "lion_di_i": item N on container page N occupies N bytes, N more than its N bytes need (at most N bytes of slack) (1 row) SELECT lion_di_try($$SELECT lion_index_verify('lion_di_i')$$); lion_di_try --------------------------------------------------------------------------------------------------------------------------------- XX002: lion index "lion_di_i": container N on block N occupies N bytes, N more than its N bytes need (at most N bytes of slack) (1 row) -- key 1 is on other pages, and the INSERT left nothing behind SELECT count(*) FROM lion_di WHERE k = 1; count ------- 54240 (1 row) DROP INDEX lion_di_i; TRUNCATE lion_di; -- ---------- 2. a line pointer within its slack that reaches into the next item ---------- -- 224 heap blocks: the fourth ARRAY is the last 32 blocks' 452 members, 912 -- bytes, right below the third, and the next row of key 7 goes into it. Its -- line pointer now claims 976 bytes - 64 of slack, which readers accept and -- never look at - of which the last 64 are the third container's. INSERT INTO lion_di SELECT CASE WHEN i % 16 = 0 THEN 7 ELSE 1 END FROM generate_series(0, 3 * 14464 + 7232 - 1) i; CREATE INDEX lion_di_i ON lion_di USING lion (k); SELECT lion_di_evict('lion_di_i'); lion_di_evict --------------- (1 row) SELECT lion_index_posting_root('lion_di_i', 7) AS lion_di_root \gset SELECT lion_di_lplen(p, 4) AS lp_len, lion_di_item(p, 3) - lion_di_item(p, 4) AS next_item_at FROM lion_di_block('lion_di_i', :lion_di_root) p; lp_len | next_item_at --------+-------------- 912 | 912 (1 row) SELECT lion_di_set_lplen('lion_di_i', :lion_di_root, 4, 976); lion_di_set_lplen ------------------- (1 row) SELECT count(*) FROM lion_di WHERE k = 7; count ------- 3164 (1 row) -- the in-place insert of the member into the fourth container SELECT lion_di_try('INSERT INTO lion_di VALUES (7)'); lion_di_try ----------------------------------------------------------------- XX002: lion index "lion_di_i": items N and N on block N overlap (1 row) -- the third container is as it was SELECT count(*) FROM lion_di WHERE k = 7; count ------- 3164 (1 row) SELECT lion_di_try($$SELECT lion_index_verify('lion_di_i')$$); lion_di_try ------------------------------------------------------------------------ XX002: lion index "lion_di_i": item N on block N overlaps another item (1 row) DROP INDEX lion_di_i; -- ---------- 3. a stored key whose header is an INDIRECT TOAST pointer's ---------- -- The two bytes 01 01 at the key of the entry the first probe of leaf 1 -- compares against, right after its 12-byte header. CREATE TABLE lion_di_t (t text NOT NULL); INSERT INTO lion_di_t SELECT md5((i % 50)::text) FROM generate_series(1, 5000) i; CREATE INDEX lion_di_t_i ON lion_di_t USING lion (t); SELECT lion_di_evict('lion_di_t_i'); lion_di_evict --------------- (1 row) SELECT lion_di_poke('lion_di_t_i', 1, lion_di_item(p, (2 + (n - 1) / 2)::int) + 12, '\x0101'::bytea) FROM lion_di_block('lion_di_t_i', 1) p, LATERAL (SELECT ((lion_di_u32(p, 12) & 65535) - 24) / 4 AS n) s; lion_di_poke -------------- (1 row) SELECT lion_di_try($$SELECT count(*) FROM lion_di_t WHERE t = md5('7')$$); lion_di_try ------------------------------------------------------------------------ XX002: lion index entry of key column N has a malformed key of N bytes (1 row) SELECT lion_di_try($$SELECT count(*) FROM lion_di_t WHERE t > md5('7')$$); lion_di_try ------------------------------------------------------------------------ XX002: lion index entry of key column N has a malformed key of N bytes (1 row) SELECT lion_di_try($$SELECT t, count(*) FROM lion_di_t GROUP BY t ORDER BY t$$); lion_di_try ------------------------------------------------------------------------ XX002: lion index entry of key column N has a malformed key of N bytes (1 row) SELECT lion_di_try($$INSERT INTO lion_di_t VALUES (md5('7'))$$); lion_di_try ------------------------------------------------------------------------ XX002: lion index entry of key column N has a malformed key of N bytes (1 row) SELECT lion_di_try($$SELECT lion_index_verify('lion_di_t_i')$$); lion_di_try -------------------------------------------------------------------------------------------- XX002: lion index "lion_di_t_i": entry N on block N has a malformed varlena key of N bytes (1 row) DROP TABLE lion_di_t; -- ---------- 4. a member at offset 0 of its heap block ---------- -- Key 7's ARRAY, on a container page, starts with (0,1), which is member 1; -- it becomes member 0, (0,0), which still sorts first. TRUNCATE lion_di; INSERT INTO lion_di SELECT CASE WHEN i % 16 = 0 THEN 7 ELSE 1 END FROM generate_series(0, 14464 - 1) i; CREATE INDEX lion_di_i ON lion_di USING lion (k) WITH (inline_limit = 64); SELECT lion_index_verify('lion_di_i', true); lion_index_verify ------------------- (1 row) SELECT lion_di_evict('lion_di_i'); lion_di_evict --------------- (1 row) SELECT lion_index_posting_root('lion_di_i', 7) AS lion_di_root \gset SELECT get_byte(p, lion_di_item(p, 1) + 8) AS first_member FROM lion_di_block('lion_di_i', :lion_di_root) p; first_member -------------- 1 (1 row) SELECT lion_di_poke('lion_di_i', :lion_di_root, lion_di_item(lion_di_block('lion_di_i', :lion_di_root), 1) + 8, '\x0000'::bytea); lion_di_poke -------------- (1 row) SELECT lion_di_try($$SELECT lion_index_verify('lion_di_i')$$); lion_di_try --------------------------------------------------------------------------------------------------------------------- XX002: lion index "lion_di_i": container N on block N is corrupt: array container member is not a heap tuple offset (1 row) SELECT lion_di_try($$SELECT lion_index_verify('lion_di_i', true)$$); lion_di_try --------------------------------------------------------------------------------------------------------------------- XX002: lion index "lion_di_i": container N on block N is corrupt: array container member is not a heap tuple offset (1 row) DROP INDEX lion_di_i; -- ---------- 5. a NARROW whose cardinality is wrong ---------- -- 55 rows to a heap page and two keys: each key's container of the first 64 -- blocks is a NARROW of width 1 and 1760 members, alone on its container -- page. Every bit of a NARROW's payload is a member it may hold, so the -- cardinality in its header (bytes 4 and 5; the type is byte 6) is all there -- is to damage but its width (section 6). The readers bound what they do by -- the payload, not by the claim, and lion_index_verify() reports it. CREATE TABLE lion_di_n (k int NOT NULL, pad text); INSERT INTO lion_di_n SELECT i % 2, repeat('x', 100) FROM generate_series(0, 64 * 55 - 1) i; CREATE INDEX lion_di_n_i ON lion_di_n USING lion (k) WITH (inline_limit = 64); SELECT narrow_containers FROM lion_index_stats('lion_di_n_i'); narrow_containers ------------------- 2 (1 row) SELECT lion_di_evict('lion_di_n_i'); lion_di_evict --------------- (1 row) SELECT lion_index_posting_root('lion_di_n_i', 1) AS lion_di_root \gset SELECT get_byte(p, lion_di_item(p, 1) + 6) AS type, get_byte(p, lion_di_item(p, 1) + 4) | (get_byte(p, lion_di_item(p, 1) + 5) << 8) AS cardinality FROM lion_di_block('lion_di_n_i', :lion_di_root) p; type | cardinality ------+------------- 5 | 1760 (1 row) SELECT lion_di_poke('lion_di_n_i', :lion_di_root, lion_di_item(lion_di_block('lion_di_n_i', :lion_di_root), 1) + 4, '\x0100'::bytea); lion_di_poke -------------- (1 row) SELECT lion_di_try('SELECT count(*) FROM lion_di_n WHERE k = 1'); lion_di_try ------------- ok (1 row) SELECT lion_di_try('SELECT k, count(*) FROM lion_di_n GROUP BY k'); lion_di_try ------------- ok (1 row) SELECT lion_di_try($$SELECT lion_index_count('lion_di_n_i', 1)$$); lion_di_try ------------- ok (1 row) SET pg_lion.enable_count_pushdown = off; SELECT count(*) FROM lion_di_n WHERE k = 1; count ------- 1760 (1 row) RESET pg_lion.enable_count_pushdown; SELECT lion_di_try($$SELECT lion_index_verify('lion_di_n_i')$$); lion_di_try ----------------------------------------------------------------------------------------------------------------------------- XX002: lion index "lion_di_n_i": container N on block N is corrupt: narrow container cardinality does not match its payload (1 row) DROP INDEX lion_di_n_i; -- ---------- 6. a NARROW of a width the build does not have ---------- -- The container of section 5 with its width, its header's flags byte (byte -- 7), made 0 and then 6: the width sizes a NARROW, so the readers refuse -- one of a width outside 1 .. 5 before they read its payload, and -- lion_index_verify() reports it. Made 2, the width sizes the item past its -- line pointer, which the readers refuse as they refuse any item longer -- than its line pointer. CREATE INDEX lion_di_n_i ON lion_di_n USING lion (k) WITH (inline_limit = 64); SELECT lion_di_evict('lion_di_n_i'); lion_di_evict --------------- (1 row) SELECT lion_index_posting_root('lion_di_n_i', 1) AS lion_di_root \gset SELECT get_byte(p, lion_di_item(p, 1) + 6) AS type, get_byte(p, lion_di_item(p, 1) + 7) AS width, lion_di_lplen(p, 1) AS lp_len FROM lion_di_block('lion_di_n_i', :lion_di_root) p; type | width | lp_len ------+-------+-------- 5 | 1 | 520 (1 row) SELECT lion_di_poke('lion_di_n_i', :lion_di_root, lion_di_item(lion_di_block('lion_di_n_i', :lion_di_root), 1) + 7, '\x00'::bytea); lion_di_poke -------------- (1 row) SELECT lion_di_try('SELECT count(*) FROM lion_di_n WHERE k = 1'); lion_di_try ------------------------------------------------------------------------------------------------------- XX002: lion index "lion_di_n_i": item N on container page N is a NARROW of width N, not one of N .. N (1 row) SELECT lion_di_try('SELECT k, count(*) FROM lion_di_n GROUP BY k'); lion_di_try ------------------------------------------------------------------------------------------------------- XX002: lion index "lion_di_n_i": item N on container page N is a NARROW of width N, not one of N .. N (1 row) SELECT lion_di_try($$SELECT lion_index_count('lion_di_n_i', 1)$$); lion_di_try ------------------------------------------------------------------------------------------------------- XX002: lion index "lion_di_n_i": item N on container page N is a NARROW of width N, not one of N .. N (1 row) SELECT lion_di_try($$SELECT lion_index_verify('lion_di_n_i')$$); lion_di_try ------------------------------------------------------------------------------------------------------------------------------- XX002: lion index "lion_di_n_i": container N on block N is corrupt: narrow container width is out of range for the block size (1 row) SELECT lion_di_evict('lion_di_n_i'); lion_di_evict --------------- (1 row) SELECT lion_di_poke('lion_di_n_i', :lion_di_root, lion_di_item(lion_di_block('lion_di_n_i', :lion_di_root), 1) + 7, '\x06'::bytea); lion_di_poke -------------- (1 row) SELECT lion_di_try('SELECT count(*) FROM lion_di_n WHERE k = 1'); lion_di_try ------------------------------------------------------------------------------------------------------- XX002: lion index "lion_di_n_i": item N on container page N is a NARROW of width N, not one of N .. N (1 row) SELECT lion_di_try($$SELECT lion_index_count('lion_di_n_i', 1)$$); lion_di_try ------------------------------------------------------------------------------------------------------- XX002: lion index "lion_di_n_i": item N on container page N is a NARROW of width N, not one of N .. N (1 row) SELECT lion_di_try($$SELECT lion_index_verify('lion_di_n_i')$$); lion_di_try ------------------------------------------------------------------------------------------------------------------------------- XX002: lion index "lion_di_n_i": container N on block N is corrupt: narrow container width is out of range for the block size (1 row) SELECT lion_di_evict('lion_di_n_i'); lion_di_evict --------------- (1 row) SELECT lion_di_poke('lion_di_n_i', :lion_di_root, lion_di_item(lion_di_block('lion_di_n_i', :lion_di_root), 1) + 7, '\x02'::bytea); lion_di_poke -------------- (1 row) SELECT lion_di_try('SELECT count(*) FROM lion_di_n WHERE k = 1'); lion_di_try -------------------------------------------------------------------------------------- XX002: lion index "lion_di_n_i": item N on container page N needs N bytes, but has N (1 row) SELECT lion_di_try($$SELECT lion_index_count('lion_di_n_i', 1)$$); lion_di_try -------------------------------------------------------------------------------------- XX002: lion index "lion_di_n_i": item N on container page N needs N bytes, but has N (1 row) SELECT lion_di_try($$SELECT lion_index_verify('lion_di_n_i')$$); lion_di_try -------------------------------------------------------------------------------------------------------------------------- XX002: lion index "lion_di_n_i": container N on block N is corrupt: narrow container does not fit in the available space (1 row) DROP TABLE lion_di_n; -- ---------- 7. a directory entry whose key length runs past its item ---------- -- The entry the first probe of leaf 1 compares against claims a key of 65535 -- bytes (keylen, bytes 2 and 3 of the entry). Every comparison and key -- decode reads keylen bytes after the header, so readers check it against -- the item's length before looking (lion_entry_at()). CREATE TABLE lion_di_t (t text NOT NULL); INSERT INTO lion_di_t SELECT md5((i % 50)::text) FROM generate_series(1, 5000) i; CREATE INDEX lion_di_t_i ON lion_di_t USING lion (t); SELECT lion_di_evict('lion_di_t_i'); lion_di_evict --------------- (1 row) SELECT lion_di_poke('lion_di_t_i', 1, lion_di_item(p, (2 + (n - 1) / 2)::int) + 2, '\xffff'::bytea) FROM lion_di_block('lion_di_t_i', 1) p, LATERAL (SELECT ((lion_di_u32(p, 12) & 65535) - 24) / 4 AS n) s; lion_di_poke -------------- (1 row) SELECT lion_di_try($$SELECT count(*) FROM lion_di_t WHERE t = md5('7')$$); lion_di_try ------------------------------------------------------------ XX002: lion index entry of N bytes claims a key of N bytes (1 row) SELECT lion_di_try($$SELECT count(*) FROM lion_di_t WHERE t > md5('7')$$); lion_di_try ------------------------------------------------------------ XX002: lion index entry of N bytes claims a key of N bytes (1 row) SELECT lion_di_try($$SELECT t, count(*) FROM lion_di_t GROUP BY t ORDER BY t$$); lion_di_try ------------------------------------------------------------ XX002: lion index entry of N bytes claims a key of N bytes (1 row) SELECT lion_di_try($$INSERT INTO lion_di_t VALUES (md5('7'))$$); lion_di_try ------------------------------------------------------------ XX002: lion index entry of N bytes claims a key of N bytes (1 row) SELECT lion_di_try($$SELECT lion_index_verify('lion_di_t_i')$$); lion_di_try --------------------------------------------------------------------------------------------- XX002: lion index "lion_di_t_i": item N on block N is N bytes, too small for its N byte key (1 row) DROP TABLE lion_di_t; -- ---------- 8. an entry that counts fewer TIDs than VACUUM removes ---------- -- Key 7's CHAIN entry on its leaf says it has one TID (ntids, bytes 16 to 23 of -- the entry; its key, a by-value int4, is the 4 bytes after its 28-byte -- header). -- VACUUM subtracting the hundreds it removes wrapped the count and wrote it -- (an assertion only); now it refuses before its record begins. CREATE TABLE lion_di_v (k int NOT NULL) WITH (autovacuum_enabled = off); INSERT INTO lion_di_v SELECT CASE WHEN i % 16 = 0 THEN 7 ELSE 1 END FROM generate_series(0, 4 * 14464 - 1) i; CREATE INDEX lion_di_v_i ON lion_di_v USING lion (k); SELECT lion_di_evict('lion_di_v_i'); lion_di_evict --------------- (1 row) -- the entry: the one CHAIN item of the index with a 4-byte key reading 7 SELECT blk AS lion_di_blk, it AS lion_di_at FROM generate_series(0, (pg_relation_size('lion_di_v_i') / 8192 - 1)::int) blk, LATERAL lion_di_block('lion_di_v_i', blk) p, LATERAL generate_series(1, GREATEST(0, LEAST(200, (((lion_di_u32(p, 12) & 65535) - 24) / 4)::int))) o, LATERAL (SELECT lion_di_item(p, o) AS it) y WHERE it BETWEEN 24 AND 8100 AND get_byte(p, it) & 2 = 2 AND get_byte(p, it + 2) = 4 AND get_byte(p, it + 3) = 0 AND substring(p FROM it + 29 FOR 4) = '\x07000000'::bytea \gset SELECT lion_di_poke('lion_di_v_i', :lion_di_blk, :lion_di_at + 16, '\x0100000000000000'::bytea); lion_di_poke -------------- (1 row) DELETE FROM lion_di_v WHERE k = 7; \set VERBOSITY terse VACUUM lion_di_v; ERROR: lion index "lion_di_v_i": entry 2 on block 3 counts 1 TIDs in 4 containers, fewer than VACUUM removes (3616 TIDs, 4 containers) \set VERBOSITY default DROP TABLE lion_di_v; -- ---------- 9. an entry too short for its positions extension ---------- -- An INLINE entry of an index that stores positions (DESIGN.md §17): its -- 12-byte header, its key, the 8-byte positions extension at the next 4-byte -- boundary, and then its payload. Its line pointer is cut to the end of its -- key: long enough for the header and the key, which is all a check of the -- key length sees, but the extension would be read past the item and the -- payload length computed from it wraps around zero. The entry is refused -- as corrupt. CREATE TABLE lion_di_s (d tsvector NOT NULL); INSERT INTO lion_di_s SELECT to_tsvector('simple', 'k' || lpad((i % 50)::text, 3, '0')) FROM generate_series(1, 5000) i; CREATE INDEX lion_di_s_i ON lion_di_s USING lion (d) WITH (store_positions = true); SELECT lion_di_evict('lion_di_s_i'); lion_di_evict --------------- (1 row) -- the INLINE entry of the lexeme k007 (INLINE is bit 0 of byte 0, POSITIONS -- bit 1 of byte 1, the column bytes 4 and 5), and the end of its key SELECT blk AS lion_di_blk, o AS lion_di_off, lion_di_lplen(p, o) AS lion_di_len, 12 + get_byte(p, it + 2) AS lion_di_keyend FROM generate_series(0, (pg_relation_size('lion_di_s_i') / 8192 - 1)::int) blk, LATERAL lion_di_block('lion_di_s_i', blk) p, LATERAL generate_series(1, GREATEST(0, LEAST(200, (((lion_di_u32(p, 12) & 65535) - 24) / 4)::int))) o, LATERAL (SELECT lion_di_item(p, o) AS it) y WHERE it BETWEEN 24 AND 8100 AND get_byte(p, it) & 1 = 1 AND get_byte(p, it + 1) & 2 = 2 AND get_byte(p, it + 3) = 0 AND get_byte(p, it + 4) = 1 AND get_byte(p, it + 5) = 0 AND position('\x6b303037'::bytea IN substring(p FROM it + 13 FOR get_byte(p, it + 2))) > 0 \gset SELECT :lion_di_len >= :lion_di_keyend + 8 AS holds_its_extension; holds_its_extension --------------------- t (1 row) SELECT lion_di_set_lplen('lion_di_s_i', :lion_di_blk, :lion_di_off, :lion_di_keyend); lion_di_set_lplen ------------------- (1 row) SELECT lion_di_try($$SELECT count(*) FROM lion_di_s WHERE d @@ 'k007'$$); lion_di_try -------------------------------------------------------------------------------------------------------------- XX002: lion index entry of N bytes at page offset N is too small for its header, key and positions extension (1 row) SELECT lion_di_try($$SELECT lion_index_verify('lion_di_s_i')$$); lion_di_try --------------------------------------------------------------------------------------------- XX002: lion index "lion_di_s_i": item N on block N is N bytes, too small for its N byte key (1 row) DROP TABLE lion_di_s; -- ---------- 10. an index of another format version ---------- -- An index with its meta page's version (at byte 28) made 0, 2, 14 and 15, -- versions this build does not write: an index of another format may hold -- what this build would read wrongly, so lion_index_verify() reports it and -- a reader refuses it, asking for a REINDEX - which makes it whole again. CREATE FUNCTION lion_di_hint(q text) RETURNS text LANGUAGE plpgsql AS $$ DECLARE h text; BEGIN EXECUTE q; RETURN 'read it'; EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS h = PG_EXCEPTION_HINT; RETURN SQLSTATE || ': ' || h; END $$; CREATE INDEX lion_di_old_i ON lion_di USING lion (k); SELECT lion_di_evict('lion_di_old_i'); lion_di_evict --------------- (1 row) SELECT lion_di_u32(lion_di_block('lion_di_old_i', 0), 28) AS version; version --------- 1 (1 row) SELECT lion_di_poke('lion_di_old_i', 0, 28, lion_di_le32(0)); lion_di_poke -------------- (1 row) SELECT lion_di_try($$SELECT lion_index_verify('lion_di_old_i')$$); lion_di_try -------------------------------------------------------- XX002: index "lion_di_old_i" is not a valid lion index (1 row) SELECT lion_di_hint($$SELECT lion_index_count('lion_di_old_i', 1)$$); lion_di_hint ------------------------------------------------------------------------------------------------------------- XX002: The index was written by a build of pg_lion with another on-disk format: REINDEX it with this build. (1 row) SELECT lion_di_evict('lion_di_old_i'); lion_di_evict --------------- (1 row) SELECT lion_di_poke('lion_di_old_i', 0, 28, lion_di_le32(2)); lion_di_poke -------------- (1 row) SELECT lion_di_try($$SELECT lion_index_verify('lion_di_old_i')$$); lion_di_try -------------------------------------------------------- XX002: index "lion_di_old_i" is not a valid lion index (1 row) SELECT lion_di_hint($$SELECT lion_index_count('lion_di_old_i', 1)$$); lion_di_hint ------------------------------------------------------------------------------------------------------------- XX002: The index was written by a build of pg_lion with another on-disk format: REINDEX it with this build. (1 row) SELECT lion_di_evict('lion_di_old_i'); lion_di_evict --------------- (1 row) SELECT lion_di_poke('lion_di_old_i', 0, 28, lion_di_le32(14)); lion_di_poke -------------- (1 row) SELECT lion_di_try($$SELECT lion_index_verify('lion_di_old_i')$$); lion_di_try -------------------------------------------------------- XX002: index "lion_di_old_i" is not a valid lion index (1 row) SELECT lion_di_hint($$SELECT lion_index_count('lion_di_old_i', 1)$$); lion_di_hint ------------------------------------------------------------------------------------------------------------- XX002: The index was written by a build of pg_lion with another on-disk format: REINDEX it with this build. (1 row) SELECT lion_di_evict('lion_di_old_i'); lion_di_evict --------------- (1 row) SELECT lion_di_poke('lion_di_old_i', 0, 28, lion_di_le32(15)); lion_di_poke -------------- (1 row) SELECT lion_di_try($$SELECT lion_index_verify('lion_di_old_i')$$); lion_di_try -------------------------------------------------------- XX002: index "lion_di_old_i" is not a valid lion index (1 row) SELECT lion_di_hint($$SELECT lion_index_count('lion_di_old_i', 1)$$); lion_di_hint ------------------------------------------------------------------------------------------------------------- XX002: The index was written by a build of pg_lion with another on-disk format: REINDEX it with this build. (1 row) REINDEX INDEX lion_di_old_i; SELECT lion_di_u32(lion_di_block('lion_di_old_i', 0), 28) AS version; version --------- 1 (1 row) SELECT lion_di_try($$SELECT lion_index_verify('lion_di_old_i')$$); lion_di_try ------------- ok (1 row) -- A backend that has read the index - this one, now, by a count - keeps -- its state, meta page words and all, and does not read the meta page -- again to use it (lion_get_index_state()). The statistics and VACUUM -- read the page for what they count, and refuse a version changed under -- that state there (lion_meta_check_version()). SELECT lion_index_count('lion_di_old_i', 1) > 0 AS warm; warm ------ t (1 row) SELECT lion_di_evict('lion_di_old_i'); lion_di_evict --------------- (1 row) SELECT lion_di_poke('lion_di_old_i', 0, 28, lion_di_le32(2)); lion_di_poke -------------- (1 row) SELECT lion_di_hint($$SELECT ntids FROM lion_index_stats('lion_di_old_i')$$); lion_di_hint ------------------------------------------------------------------------------------------------------------- XX002: The index was written by a build of pg_lion with another on-disk format: REINDEX it with this build. (1 row) VACUUM lion_di; ERROR: index "lion_di_old_i" is not a valid lion index DETAIL: Meta page magic 52424931 version 2, expected 52424931 version 1. HINT: The index was written by a build of pg_lion with another on-disk format: REINDEX it with this build. CONTEXT: while cleaning up index "lion_di_old_i" of relation "public.lion_di" -- To the planner such an index is not there (DESIGN.md §21, "An index of -- another format"): it reads lion indexes for its estimates whether or not -- the plan uses them, and every query that so much as estimated the column -- failed with the index's error. A new backend - one whose relcache holds -- no state of the index, as every backend after an upgrade - warns of it -- once and plans the table's queries without it; an insert still refuses -- it, and a REINDEX gives it back to the planner. CREATE FUNCTION lion_di_lion(q text) RETURNS boolean LANGUAGE plpgsql AS $$ DECLARE ln text; lion boolean := false; BEGIN FOR ln IN EXECUTE 'EXPLAIN (COSTS OFF) ' || q LOOP lion := lion OR ln ~ '(Lion|lion_di_old_i)'; END LOOP; RETURN lion; END $$; \c SET statement_timeout = '60s'; SELECT lion_di_lion('SELECT count(*) FROM lion_di WHERE k = 7') AS planned_with_lion; WARNING: lion index "lion_di_old_i" is of another on-disk format, and is not used HINT: REINDEX the index. Until then the planner leaves it out, and inserts into its table and VACUUM fail. planned_with_lion ------------------- f (1 row) SELECT count(*) FROM lion_di WHERE k = 7; count ------- 904 (1 row) SELECT k, count(*) FROM lion_di GROUP BY k ORDER BY k; k | count ---+------- 1 | 13560 7 | 904 (2 rows) SELECT count(*) FROM lion_di a JOIN lion_di b ON a.k = b.k WHERE a.k = 7 AND b.k = 7; count -------- 817216 (1 row) INSERT INTO lion_di VALUES (7); ERROR: index "lion_di_old_i" is not a valid lion index DETAIL: Meta page magic 52424931 version 2, expected 52424931 version 1. HINT: The index was written by a build of pg_lion with another on-disk format: REINDEX it with this build. REINDEX INDEX lion_di_old_i; SELECT lion_di_lion('SELECT count(*) FROM lion_di WHERE k = 7') AS planned_with_lion; planned_with_lion ------------------- t (1 row) SELECT count(*) FROM lion_di WHERE k = 7; count ------- 904 (1 row) DROP FUNCTION lion_di_lion(text); DROP INDEX lion_di_old_i; DROP FUNCTION lion_di_hint(text); -- ---------- 11. an entry whose line pointer reaches into a CHAIN entry ---------- -- Key 7's posting set is a chain, as in 1, and its CHAIN entry shares a leaf -- with the entries of key 1 and of keys 100 to 149, five rows each. The -- entry whose bytes end where key 7's begin has its line pointer stretched -- over the first 24 of them, which hold the counters (ntids is bytes 16 to -- 23). Readers take the stretched entry: its key is where it was. The -- insert of a row of key 7 rewrites the CHAIN entry, counters and all, in -- the record that adds the member, and is refused before that record -- begins. CREATE TABLE lion_di_c (k int NOT NULL) WITH (autovacuum_enabled = off); INSERT INTO lion_di_c SELECT CASE WHEN i % 16 = 0 THEN 7 WHEN i < 250 THEN 100 + i % 50 ELSE 1 END FROM generate_series(0, 4 * 14464 - 1) i; CREATE INDEX lion_di_c_i ON lion_di_c USING lion (k); SELECT lion_di_evict('lion_di_c_i'); lion_di_evict --------------- (1 row) -- the CHAIN entry of key 7, as in 8, and the entry right below it on the page SELECT blk AS lion_di_blk, o AS lion_di_off, it AS lion_di_at FROM generate_series(0, (pg_relation_size('lion_di_c_i') / 8192 - 1)::int) blk, LATERAL lion_di_block('lion_di_c_i', blk) p, LATERAL generate_series(1, GREATEST(0, LEAST(200, (((lion_di_u32(p, 12) & 65535) - 24) / 4)::int))) o, LATERAL (SELECT lion_di_item(p, o) AS it) y WHERE it BETWEEN 24 AND 8100 AND get_byte(p, it) & 2 = 2 AND get_byte(p, it + 2) = 4 AND get_byte(p, it + 3) = 0 AND substring(p FROM it + 29 FOR 4) = '\x07000000'::bytea \gset SELECT o AS lion_di_below, :lion_di_at - it + 24 AS lion_di_len FROM lion_di_block('lion_di_c_i', :lion_di_blk) p, LATERAL generate_series(1, (((lion_di_u32(p, 12) & 65535) - 24) / 4)::int) o, LATERAL (SELECT lion_di_item(p, o) AS it) y WHERE o <> :lion_di_off AND lion_di_lplen(p, o) > 0 AND it + ((lion_di_lplen(p, o) + 7) / 8) * 8 = :lion_di_at \gset SELECT lion_di_set_lplen('lion_di_c_i', :lion_di_blk, :lion_di_below, :lion_di_len); lion_di_set_lplen ------------------- (1 row) SELECT count(*) FROM lion_di_c WHERE k = 7; count ------- 3616 (1 row) SELECT lion_di_try('INSERT INTO lion_di_c VALUES (7)'); lion_di_try ------------------------------------------------------------------- XX002: lion index "lion_di_c_i": items N and N on block N overlap (1 row) SELECT count(*) FROM lion_di_c WHERE k = 7; count ------- 3616 (1 row) SELECT lion_di_try($$SELECT lion_index_verify('lion_di_c_i')$$); lion_di_try -------------------------------------------------------------------------- XX002: lion index "lion_di_c_i": item N on block N overlaps another item (1 row) DROP TABLE lion_di_c; -- ---------- 12. position chunks at the same bytes, and VACUUM ---------- -- 'every' is twice in every row, so its positions spill to a position tree. -- The second line pointer of one of its leaves is made a copy of the -- first: each still reads as a whole chunk, but VACUUM writes the chunks -- back by their line pointers in one record, so the page is refused first. CREATE TABLE lion_di_p (id int, d tsvector); INSERT INTO lion_di_p SELECT g, to_tsvector('simple', 'every word every') FROM generate_series(1, 4000) g; CREATE INDEX lion_di_p_i ON lion_di_p USING lion (d) WITH (store_positions = true); SELECT lion_di_evict('lion_di_p_i'); lion_di_evict --------------- (1 row) SELECT min(blk) AS lion_di_blk FROM generate_series(1, (pg_relation_size('lion_di_p_i') / 8192 - 1)::int) blk, LATERAL lion_di_block('lion_di_p_i', blk) p, LATERAL (SELECT (lion_di_u32(p, 16) & 65535)::int AS sp) s WHERE sp BETWEEN 24 AND 8160 AND (lion_di_u32(p, sp + 28) & 65535) = 65415 -- LION_PAGE_ID AND (lion_di_u32(p, sp + 26) & 140) = 132 -- CONTAINER | POSITIONS, not DELETED AND (lion_di_u32(p, 12) & 65535) >= 32 \gset SELECT lion_di_poke('lion_di_p_i', :lion_di_blk, 28, substring(lion_di_block('lion_di_p_i', :lion_di_blk) FROM 25 FOR 4)); lion_di_poke -------------- (1 row) DELETE FROM lion_di_p WHERE id = 1; \set VERBOSITY terse VACUUM lion_di_p; ERROR: lion index "lion_di_p_i": item 2 on block 1 overlaps another item \set VERBOSITY default DROP TABLE lion_di_p; -- ---------- an undamaged index still works ---------- CREATE INDEX lion_di_i ON lion_di USING lion (k); SELECT count(*) FROM lion_di WHERE k = 7; count ------- 904 (1 row) SELECT lion_index_verify('lion_di_i'); lion_index_verify ------------------- (1 row) RESET enable_seqscan; RESET statement_timeout; DROP TABLE lion_di; DROP FUNCTION lion_di_try(text); DROP FUNCTION lion_di_set_lplen(regclass, bigint, int, int); DROP FUNCTION lion_di_poke(regclass, bigint, int, bytea); DROP FUNCTION lion_di_evict(regclass); DROP FUNCTION lion_di_lplen(bytea, int); DROP FUNCTION lion_di_item(bytea, int); DROP FUNCTION lion_di_block(regclass, bigint); DROP FUNCTION lion_di_le32(bigint); DROP FUNCTION lion_di_u32(bytea, int); \else \echo 'pg_buffercache_evict() needs PostgreSQL 17: skipped' \endif