-- Stored positions (DESIGN.md §17, "Stored positions"): insert, VACUUM,
-- CREATE INDEX, and the bitmap scans they answer.
--
-- A column whose multi-key opclass has support function 5 (tsvector_ops), in
-- an index built WITH (store_positions = true), stores for every key each
-- row's word positions and weights: in the entry while it is INLINE, in a
-- position tree once the entry spills.  The test creates a function that
-- lists what the index stores for one key, and compares that with unnest() of
-- the rows' tsvectors.
--
-- The table keeps autovacuum off so that the VACUUMs below are the only ones.
\set VERBOSITY terse
SET client_min_messages = warning;
-- a heap page becomes all-visible only once its rows' commits are flushed,
-- and the dev cluster commits asynchronously (count.sql says more)
SET synchronous_commit = on;
CREATE EXTENSION IF NOT EXISTS pg_lion;
CREATE FUNCTION lion_debug_key_positions(regclass, text,
	OUT tid tid, OUT positions int2[], OUT weights text[]) RETURNS SETOF record
	AS '$libdir/pg_lion' LANGUAGE C STRICT;
CREATE TABLE pos_docs (id int, d tsvector) WITH (autovacuum_enabled = off);
CREATE INDEX pos_docs_d ON pos_docs USING lion (d) WITH (store_positions = true);
-- What the index stores for every lexeme of the table, live rows only (an
-- UPDATE leaves the old row's positions behind, for VACUUM), against the
-- rows themselves.
CREATE VIEW pos_compare AS
WITH idx AS (
	SELECT l.lexeme, p.tid, p.positions, p.weights
	FROM (SELECT DISTINCT unnest(tsvector_to_array(d)) AS lexeme
		  FROM pos_docs) l,
		 lion_debug_key_positions('pos_docs_d', l.lexeme) p
	WHERE p.tid IN (SELECT ctid FROM pos_docs)),
heap AS (
	SELECT u.lexeme, pos_docs.ctid AS tid, u.positions, u.weights
	FROM pos_docs, unnest(d) u)
SELECT (SELECT count(*) FROM heap) AS heap_rows,
	   (SELECT count(*) FROM (SELECT * FROM idx EXCEPT ALL SELECT * FROM heap) x) AS only_index,
	   (SELECT count(*) FROM (SELECT * FROM heap EXCEPT ALL SELECT * FROM idx) x) AS only_heap;
-- one row, then a few: inline entries
INSERT INTO pos_docs VALUES (1, 'a:1A b:2,5 c:3B');
INSERT INTO pos_docs VALUES (2, 'a:7 c:1C,2,9D');
INSERT INTO pos_docs VALUES (3, strip('a:1 b:2'::tsvector));	-- no positions
INSERT INTO pos_docs VALUES (4, ''::tsvector);				-- no lexemes
SELECT * FROM lion_debug_key_positions('pos_docs_d', 'a') ORDER BY tid;
  tid  | positions | weights 
-------+-----------+---------
 (0,1) | {1}       | {A}
 (0,2) | {7}       | {D}
 (0,3) |           | 
(3 rows)

SELECT * FROM lion_debug_key_positions('pos_docs_d', 'b') ORDER BY tid;
  tid  | positions | weights 
-------+-----------+---------
 (0,1) | {2,5}     | {D,D}
 (0,3) |           | 
(2 rows)

SELECT * FROM pos_compare;
 heap_rows | only_index | only_heap 
-----------+------------+-----------
         7 |          0 |         0
(1 row)

-- positions at the limits: a tsvector keeps at most 255 per lexeme, and the
-- largest position is 16383
SELECT array_length(positions, 1) AS npos, positions[256] AS last
FROM (SELECT string_agg('long', ' ') AS t FROM generate_series(1, 300)) s,
	 unnest(to_tsvector('simple', t));
 npos | last 
------+------
  255 |     
(1 row)

INSERT INTO pos_docs
SELECT 5, to_tsvector('simple', string_agg('long', ' ')) FROM generate_series(1, 300);
INSERT INTO pos_docs VALUES (6, 'edge:16383A');
SELECT tid, array_length(positions, 1) AS npos, positions[array_length(positions, 1)] AS last
FROM lion_debug_key_positions('pos_docs_d', 'long');
  tid  | npos | last 
-------+------+------
 (0,5) |  255 |  255
(1 row)

SELECT * FROM lion_debug_key_positions('pos_docs_d', 'edge');
  tid  | positions | weights 
-------+-----------+---------
 (0,6) | {16383}   | {A}
(1 row)

-- many rows: rare words stay inline, common ones spill to position trees,
-- and an UPDATE adds a second version of every third row
SELECT setseed(0.42);
 setseed 
---------
 
(1 row)

INSERT INTO pos_docs
SELECT g, setweight((SELECT string_agg(w, ' ')
					 FROM (SELECT 'w' || floor(random() * random() * 300)::int AS w
						   FROM generate_series(1, 5 + g % 40)) s)::tsvector,
					(ARRAY['A', 'B', 'C', 'D'])[1 + g % 4]::"char")
FROM generate_series(10, 2500) g;
UPDATE pos_docs SET d = to_tsvector('simple', 'common rare' || id || ' common again common') || d
WHERE id % 3 = 0;
SELECT * FROM pos_compare;
 heap_rows | only_index | only_heap 
-----------+------------+-----------
     58162 |          0 |         0
(1 row)

SELECT lion_index_verify('pos_docs_d');
 lion_index_verify 
-------------------
 
(1 row)

SELECT lion_index_verify('pos_docs_d', true);
 lion_index_verify 
-------------------
 
(1 row)

SELECT count(*) FROM pos_docs WHERE d @@ 'common & again';
 count 
-------
   832
(1 row)

-- VACUUM takes the dead rows' positions out with their TIDs: inline ones in
-- the entry's rewrite, spilled ones from the position tree after the posting
-- set; what is left matches the rows exactly, dead versions and all gone
DELETE FROM pos_docs WHERE id % 7 = 0;
VACUUM pos_docs;
SELECT * FROM pos_compare;
 heap_rows | only_index | only_heap 
-----------+------------+-----------
     49856 |          0 |         0
(1 row)

SELECT count(*) AS dead_left
FROM (SELECT DISTINCT unnest(tsvector_to_array(d)) AS lexeme FROM pos_docs) l,
	 lion_debug_key_positions('pos_docs_d', l.lexeme) p
WHERE p.tid NOT IN (SELECT ctid FROM pos_docs);
 dead_left 
-----------
         0
(1 row)

SELECT lion_index_verify('pos_docs_d', true);
 lion_index_verify 
-------------------
 
(1 row)

-- a key whose rows all go takes its position tree with it, and the next
-- rows of that key start a new one
DELETE FROM pos_docs WHERE d @@ 'common';
VACUUM pos_docs;
SELECT count(*) FROM lion_debug_key_positions('pos_docs_d', 'common');
 count 
-------
     0
(1 row)

INSERT INTO pos_docs
SELECT g, to_tsvector('simple', 'common again ' || g || ' common') FROM generate_series(1, 600) g;
SELECT * FROM pos_compare;
 heap_rows | only_index | only_heap 
-----------+------------+-----------
     33602 |          0 |         0
(1 row)

SELECT lion_index_verify('pos_docs_d', true);
 lion_index_verify 
-------------------
 
(1 row)

-- A build over the rows there are (REINDEX): the same positions, with keys of
-- every shape - inline ones, a few rows with many positions each (a chunk
-- too large for the entry, or more than a chunk: a CHAIN entry with a small
-- posting set), and a key in every row whose tree is more than a leaf
INSERT INTO pos_docs
SELECT 3000 + g, to_tsvector('simple', repeat('dense ', 200) || 'few' || (g % 3))
FROM generate_series(1, 12) g;
INSERT INTO pos_docs
SELECT 4000 + g, to_tsvector('simple', 'everywhere ' || repeat('again ', 1 + g % 20))
FROM generate_series(1, 4000) g;
REINDEX INDEX pos_docs_d;
SELECT * FROM pos_compare;
 heap_rows | only_index | only_heap 
-----------+------------+-----------
     41626 |          0 |         0
(1 row)

SELECT lion_index_verify('pos_docs_d', true);
 lion_index_verify 
-------------------
 
(1 row)

-- and it goes on from there: inserts, deletes, VACUUM
INSERT INTO pos_docs
SELECT 9000 + g, to_tsvector('simple', 'everywhere dense new' || g) FROM generate_series(1, 300) g;
DELETE FROM pos_docs WHERE id % 11 = 0;
VACUUM pos_docs;
SELECT * FROM pos_compare;
 heap_rows | only_index | only_heap 
-----------+------------+-----------
     38702 |          0 |         0
(1 row)

SELECT lion_index_verify('pos_docs_d', true);
 lion_index_verify 
-------------------
 
(1 row)

-- A build in the least memory there is writes the keys' positions out to a
-- file more than once and reads each key's parts back in order; heap-only
-- tuples, which the scan reports under their chain's root offset, bring a
-- page's rows to it out of order
CREATE TABLE pos_hot (id int, d tsvector)
	WITH (fillfactor = 50, autovacuum_enabled = off);
INSERT INTO pos_hot
SELECT g, to_tsvector('simple', repeat('dense ', 100 + g % 3) || 'w' || (g % 50) ||
							   repeat(' pad', g % 2) || ' hot' || (g % 3))
FROM generate_series(1, 3000) g;
CREATE INDEX pos_hot_d ON pos_hot USING lion (d) WITH (store_positions = true);
UPDATE pos_hot SET id = -id WHERE id % 4 = 0;
SELECT count(*) AS moved FROM pos_hot WHERE id < 0;
 moved 
-------
   750
(1 row)

-- the index keeps a chain's root offset, so its entries are compared with the
-- heap's by block; the phrases, whose answer differs between neighbours on a
-- page, check each row got its own positions
CREATE FUNCTION pos_hot_check(OUT heap_rows bigint, OUT only_index bigint,
							  OUT only_heap bigint, OUT phrases bigint,
							  OUT matched numeric, OUT differ bigint)
LANGUAGE plpgsql AS $$
BEGIN
	WITH idx AS (
		SELECT l.lexeme, (p.tid::text::point)[0] AS blk, p.positions, p.weights
		FROM (SELECT DISTINCT unnest(tsvector_to_array(d)) AS lexeme FROM pos_hot) l,
			 lion_debug_key_positions('pos_hot_d', l.lexeme) p),
	heap AS (
		SELECT u.lexeme, (pos_hot.ctid::text::point)[0] AS blk, u.positions, u.weights
		FROM pos_hot, unnest(d) u)
	SELECT (SELECT count(*) FROM heap),
		   (SELECT count(*) FROM (SELECT * FROM idx EXCEPT ALL SELECT * FROM heap) x),
		   (SELECT count(*) FROM (SELECT * FROM heap EXCEPT ALL SELECT * FROM idx) x)
	INTO heap_rows, only_index, only_heap;
	PERFORM set_config('enable_seqscan', 'off', true);
	CREATE TEMP TABLE pos_hot_idx AS
	SELECT format('w%s <-> hot%s', w, h)::tsquery AS q, 0::bigint AS n
	FROM generate_series(0, 49) w, generate_series(0, 2) h;
	UPDATE pos_hot_idx SET n = (SELECT count(*) FROM pos_hot WHERE d @@ q);
	PERFORM set_config('enable_seqscan', 'on', true);
	PERFORM set_config('enable_bitmapscan', 'off', true);
	PERFORM set_config('enable_indexscan', 'off', true);
	PERFORM set_config('pg_lion.enable_count_pushdown', 'off', true);
	SELECT count(*), sum(i.n),
		   count(*) FILTER (WHERE i.n IS DISTINCT FROM
			   (SELECT count(*) FROM pos_hot WHERE d @@ i.q))
	INTO phrases, matched, differ
	FROM pos_hot_idx i;
	DROP TABLE pos_hot_idx;
END $$;
SET maintenance_work_mem = '1MB';
REINDEX INDEX pos_hot_d;
RESET maintenance_work_mem;
SELECT * FROM pos_hot_check();
 heap_rows | only_index | only_heap | phrases | matched | differ 
-----------+------------+-----------+---------+---------+--------
     10500 |          0 |         0 |     150 |    1500 |      0
(1 row)

SELECT lion_index_verify('pos_hot_d', true);
 lion_index_verify 
-------------------
 
(1 row)

-- A parallel build: each participant spools the positions of its share of the
-- heap with the codes, and the leader merges a key's members by code
-- (PostgreSQL 17 and later; before, the same build runs serially)
ALTER TABLE pos_hot SET (parallel_workers = 2);
SET max_parallel_maintenance_workers = 2;
SET maintenance_work_mem = '96MB';
REINDEX INDEX pos_hot_d;
RESET maintenance_work_mem;
RESET max_parallel_maintenance_workers;
SELECT * FROM pos_hot_check();
 heap_rows | only_index | only_heap | phrases | matched | differ 
-----------+------------+-----------+---------+---------+--------
     10500 |          0 |         0 |     150 |    1500 |      0
(1 row)

SELECT lion_index_verify('pos_hot_d', true);
 lion_index_verify 
-------------------
 
(1 row)

DROP FUNCTION pos_hot_check();
DROP TABLE pos_hot;
-- Queries answered from the positions (lion_posfilter.c): the posting sets
-- give the candidates, and each one is decided from its stored positions, so
-- a bitmap scan or an index scan returns exactly what the heap's @@ does and
-- has nothing to recheck.  Every shape against a seq scan, over inline keys
-- and position trees, with stripped rows and the last position there is.
INSERT INTO pos_docs VALUES
	(20001, strip(to_tsvector('simple', 'common again common'))),
	(20002, 'near:16382 far:16383'),
	(20003, 'far:1A near:3B,16383C');
CREATE TEMP TABLE pos_queries (q tsquery);
INSERT INTO pos_queries VALUES
	('common <-> again'), ('again <-> common'), ('again <2> common'),
	('common <-> again <-> common'), ('everywhere <-> again <-> again'),
	('dense <-> dense'), ('dense <-> few1'), ('dense <200> few2'),
	('w1:A'), ('w1:B & w2'), ('w1:AB <-> w2'), ('w2:D <-> w1:CD'),
	('common & !again'), ('everywhere & !again'), ('again & !(common <-> again)'),
	('common <-> !again'), ('!common <-> again'), ('common & !nosuch'),
	('(common <-> again) | (w1 <-> w2)'), ('(common | dense) <-> again'),
	('common <-> (again | new5)'), ('near <-> far'), ('far <-> near'),
	('near:C <2> far'), ('far:A <2> near:B'), ('nosuch <-> common'),
	('w5 <-> w5'), ('rare3 <-> common'), ('common <-> ag:*');
SET enable_seqscan = off;
SET enable_indexscan = off;
CREATE TEMP TABLE pos_by_index AS
SELECT q::text, (SELECT array_agg(id ORDER BY id) FROM pos_docs WHERE d @@ q) AS ids
FROM pos_queries;
RESET enable_indexscan;
SET enable_bitmapscan = off;
CREATE TEMP TABLE pos_by_iscan AS
SELECT q::text, (SELECT array_agg(id ORDER BY id) FROM pos_docs WHERE d @@ q) AS ids
FROM pos_queries;
RESET enable_seqscan;
SET enable_indexscan = off;
CREATE TEMP TABLE pos_by_heap AS
SELECT q::text, (SELECT array_agg(id ORDER BY id) FROM pos_docs WHERE d @@ q) AS ids
FROM pos_queries;
RESET enable_bitmapscan;
RESET enable_indexscan;
SELECT i.q, coalesce(cardinality(h.ids), 0) AS rows,
	   i.ids IS NOT DISTINCT FROM h.ids AS bitmap_same,
	   s.ids IS NOT DISTINCT FROM h.ids AS iscan_same
FROM pos_by_index i JOIN pos_by_heap h USING (q) JOIN pos_by_iscan s USING (q)
ORDER BY i.q;
                  q                   | rows | bitmap_same | iscan_same 
--------------------------------------+------+-------------+------------
 !'common' <-> 'again'                | 3636 | t           | t
 'again' & !( 'common' <-> 'again' )  | 3637 | t           | t
 'again' <-> 'common'                 |    0 | t           | t
 'again' <2> 'common'                 |  546 | t           | t
 'common' & !'again'                  |    0 | t           | t
 'common' & !'nosuch'                 |  547 | t           | t
 'common' <-> !'again'                |  546 | t           | t
 'common' <-> 'ag':*                  |  546 | t           | t
 'common' <-> 'again'                 |  546 | t           | t
 'common' <-> 'again' <-> 'common'    |    0 | t           | t
 'common' <-> 'again' | 'w1' <-> 'w2' |  546 | t           | t
 'common' <-> ( 'again' | 'new5' )    |  546 | t           | t
 'dense' <-> 'dense'                  |   11 | t           | t
 'dense' <-> 'few1'                   |    4 | t           | t
 'dense' <200> 'few2'                 |    4 | t           | t
 'everywhere' & !'again'              |  273 | t           | t
 'everywhere' <-> 'again' <-> 'again' | 3454 | t           | t
 'far' <-> 'near'                     |    0 | t           | t
 'far':A <2> 'near':B                 |    1 | t           | t
 'near' <-> 'far'                     |    1 | t           | t
 'near':C <2> 'far'                   |    0 | t           | t
 'nosuch' <-> 'common'                |    0 | t           | t
 'rare3' <-> 'common'                 |    0 | t           | t
 'w1':A                               |  439 | t           | t
 'w1':AB <-> 'w2'                     |    0 | t           | t
 'w1':B & 'w2'                        |  161 | t           | t
 'w2':D <-> 'w1':CD                   |    0 | t           | t
 'w5' <-> 'w5'                        |    0 | t           | t
 ( 'common' | 'dense' ) <-> 'again'   |  546 | t           | t
(29 rows)

-- what the bitmap heap scan still rechecks: nothing, but for a prefix
-- lexeme, which names keys the filter cannot follow
CREATE FUNCTION pos_rechecked(q tsquery) RETURNS bool LANGUAGE plpgsql AS $$
DECLARE
	l text;
BEGIN
	SET LOCAL enable_seqscan = off;
	SET LOCAL enable_indexscan = off;
	FOR l IN EXECUTE format('EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) '
							'SELECT id FROM pos_docs WHERE d @@ %L', q) LOOP
		IF l LIKE '%Rows Removed by Index Recheck%' THEN
			RETURN true;
		END IF;
	END LOOP;
	RETURN false;
END $$;
SELECT q, pos_rechecked(q) FROM (VALUES
	('common <-> again'::tsquery), ('everywhere & !again'), ('w1:A'),
	('common <-> ag:*')) v(q);
            q            | pos_rechecked 
-------------------------+---------------
 'common' <-> 'again'    | f
 'everywhere' & !'again' | f
 'w1':A                  | f
 'common' <-> 'ag':*     | f
(4 rows)

DROP FUNCTION pos_rechecked(tsquery);
-- `@@ ANY (array)` in a plain index scan: the union of its elements, each
-- element's superset through a filter of its own (LION_KN_POSFILTER), so
-- the heap rechecks nothing unless an element has a prefix lexeme
CREATE TEMP TABLE pos_arrays AS
SELECT ARRAY[a.q, b.q] AS arr FROM pos_queries a, pos_queries b
WHERE a.q::text < b.q::text
UNION ALL SELECT ARRAY[q, NULL] FROM pos_queries;
SET enable_seqscan = off;
SET enable_bitmapscan = off;
CREATE TEMP TABLE pos_any_iscan AS
SELECT arr::text AS a,
	   (SELECT array_agg(id ORDER BY id) FROM pos_docs WHERE d @@ ANY (arr)) AS ids
FROM pos_arrays;
RESET enable_seqscan;
SET enable_indexscan = off;
CREATE TEMP TABLE pos_any_heap AS
SELECT arr::text AS a,
	   (SELECT array_agg(id ORDER BY id) FROM pos_docs WHERE d @@ ANY (arr)) AS ids
FROM pos_arrays;
RESET enable_bitmapscan;
RESET enable_indexscan;
SELECT count(*) AS arrays,
	   count(*) FILTER (WHERE i.ids IS DISTINCT FROM h.ids) AS differ
FROM pos_any_iscan i JOIN pos_any_heap h USING (a);
 arrays | differ 
--------+--------
    435 |      0
(1 row)

CREATE FUNCTION pos_any_explain(arr tsquery[]) RETURNS SETOF text
LANGUAGE plpgsql AS $$
DECLARE
	l text;
BEGIN
	SET LOCAL enable_seqscan = off;
	SET LOCAL enable_bitmapscan = off;
	FOR l IN EXECUTE format('EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF, BUFFERS OFF) '
							'SELECT id FROM pos_docs WHERE d @@ ANY (%L)', arr) LOOP
		CONTINUE WHEN l LIKE '%Index Searches:%';
		RETURN NEXT regexp_replace(l, ' \(actual .*\)$', '');
	END LOOP;
END $$;
SELECT pos_any_explain('{"common <-> again", "far <-> near"}');
                                       pos_any_explain                                       
---------------------------------------------------------------------------------------------
 Index Scan using pos_docs_d on pos_docs
   Index Cond: (d @@ ANY ('{"''common'' <-> ''again''","''far'' <-> ''near''"}'::tsquery[]))
(2 rows)

SELECT pos_any_explain('{"common <-> again", "common <-> ag:*"}');
                                        pos_any_explain                                         
------------------------------------------------------------------------------------------------
 Index Scan using pos_docs_d on pos_docs
   Index Cond: (d @@ ANY ('{"''common'' <-> ''again''","''common'' <-> ''ag'':*"}'::tsquery[]))
(2 rows)

DROP FUNCTION pos_any_explain(tsquery[]);
-- The count pushdown decides the same candidates from the positions before
-- it asks the visibility map, so a vacuumed table is counted without the
-- heap; a literal query and a generic plan's parameter alike
VACUUM pos_docs;
CREATE TEMP TABLE pos_counted AS
SELECT q::text, (SELECT count(*) FROM pos_docs WHERE d @@ q) AS n FROM pos_queries;
PREPARE pos_count(tsquery) AS SELECT count(*) FROM pos_docs WHERE d @@ $1;
SET plan_cache_mode = force_generic_plan;
CREATE TEMP TABLE pos_generic (q text, n bigint);
DO $$
DECLARE
	r record;
	n bigint;
BEGIN
	FOR r IN SELECT q FROM pos_queries LOOP
		EXECUTE format('EXECUTE pos_count(%L)', r.q) INTO n;
		INSERT INTO pos_generic VALUES (r.q::text, n);
	END LOOP;
END $$;
RESET plan_cache_mode;
DEALLOCATE pos_count;
SELECT c.q, c.n AS counted, g.n AS generic, coalesce(cardinality(h.ids), 0) AS heap
FROM pos_counted c JOIN pos_generic g USING (q) JOIN pos_by_heap h USING (q)
WHERE c.n <> coalesce(cardinality(h.ids), 0) OR g.n <> c.n;
 q | counted | generic | heap 
---+---------+---------+------
(0 rows)

CREATE FUNCTION pos_count_explain(q tsquery) RETURNS SETOF text LANGUAGE plpgsql AS $$
DECLARE
	l text;
BEGIN
	FOR l IN EXECUTE format('EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) '
							'SELECT count(*) FROM pos_docs WHERE d @@ %L', q) LOOP
		IF l ~ '(Custom Scan|Heap TIDs Rechecked|Position|Removed)' THEN
			RETURN NEXT regexp_replace(l, ' \(actual .*\)$', '');
		END IF;
	END LOOP;
END $$;
SELECT pos_count_explain('common <-> again');
       pos_count_explain        
--------------------------------
 Custom Scan (LionCount)
   Heap TIDs Rechecked: 0
   Position Checks: 547
   Rows Removed by Positions: 1
(4 rows)

SELECT pos_count_explain('common <-> ag:*');
       pos_count_explain        
--------------------------------
 Custom Scan (LionCount)
   Heap TIDs Rechecked: 0
   Position Checks: 547
   Rows Removed by Positions: 1
(4 rows)

DROP FUNCTION pos_count_explain(tsquery);
-- ... and per group: the walk that intersects a copy of the WHERE with every
-- group's set filters the copy once a key (lion_count_groups.c), the
-- distinct count and the join's per-key counts likewise
CREATE TABLE pos_grouped (id int, cat int, d tsvector) WITH (autovacuum_enabled = off);
INSERT INTO pos_grouped SELECT id, id % 7, d FROM pos_docs;
CREATE INDEX pos_grouped_cd ON pos_grouped USING lion (cat, d) WITH (store_positions = true);
CREATE TABLE pos_dim (id int PRIMARY KEY, name text);
INSERT INTO pos_dim SELECT g, 'n' || (g % 3) FROM generate_series(0, 6) g;
VACUUM ANALYZE pos_grouped, pos_dim;
-- The distinct count's filter reads all three lexemes, `again` in 4,183 rows
-- of 5,767 (lion_cost_posfilter()), and a sort of the rows a bitmap heap scan
-- fetches comes within a few percent of it on a table this small: sorts are
-- off here, so that the count is the one checked
SET enable_sort = off;
EXPLAIN (COSTS OFF)
SELECT cat, count(*) FROM pos_grouped WHERE d @@ 'everywhere <-> again' GROUP BY cat;
                                                QUERY PLAN                                                 
-----------------------------------------------------------------------------------------------------------
 Custom Scan (LionCount)
   Lion Indexes: pos_grouped_cd.cat (cat), pos_grouped_cd.d (d @@ '''everywhere'' <-> ''again'''::tsquery)
   Group Key: cat
(3 rows)

EXPLAIN (COSTS OFF)
SELECT count(DISTINCT cat) FROM pos_grouped WHERE d @@ 'common <-> again & !new5';
                                                    QUERY PLAN                                                     
-------------------------------------------------------------------------------------------------------------------
 Custom Scan (LionCount)
   Lion Indexes: pos_grouped_cd.cat (cat), pos_grouped_cd.d (d @@ '''common'' <-> ''again'' & !''new5'''::tsquery)
   Distinct Key: cat
(3 rows)

-- PostgreSQL 19 may aggregate below the join instead (eager aggregation)
DO $$BEGIN IF EXISTS (SELECT 1 FROM pg_settings WHERE name = 'enable_eager_aggregate') THEN PERFORM set_config('enable_eager_aggregate', 'off', false); END IF; END$$;
EXPLAIN (COSTS OFF)
SELECT pos_dim.name, count(*)
FROM pos_grouped JOIN pos_dim ON pos_grouped.cat = pos_dim.id
WHERE pos_grouped.d @@ 'w1:A | (common <-> again)' GROUP BY pos_dim.name;
                                                             QUERY PLAN                                                              
-------------------------------------------------------------------------------------------------------------------------------------
 Finalize HashAggregate
   Group Key: pos_dim.name
   ->  Custom Scan (LionCount)
         Lion Indexes: pos_grouped_cd.cat (cat = pos_dim.id), pos_grouped_cd.d (d @@ '''w1'':A | ''common'' <-> ''again'''::tsquery)
         Join Key Lookups: in index order
         Fact Filters: collected once
         ->  Seq Scan on pos_dim
(7 rows)

CREATE TEMP TABLE pos_pushed AS
SELECT 'group' AS what, cat::text AS k, count(*) AS n
FROM pos_grouped WHERE d @@ 'everywhere <-> again' GROUP BY cat
UNION ALL
SELECT 'distinct', NULL, count(DISTINCT cat)
FROM pos_grouped WHERE d @@ 'common <-> again & !new5'
UNION ALL
SELECT 'join', pos_dim.name, count(*)
FROM pos_grouped JOIN pos_dim ON pos_grouped.cat = pos_dim.id
WHERE pos_grouped.d @@ 'w1:A | (common <-> again)' GROUP BY pos_dim.name;
DO $$BEGIN IF EXISTS (SELECT 1 FROM pg_settings WHERE name = 'enable_eager_aggregate') THEN EXECUTE 'RESET enable_eager_aggregate'; END IF; END$$;
RESET enable_sort;
SET pg_lion.enable_count_pushdown = off;
SET enable_bitmapscan = off;
SET enable_indexscan = off;
SELECT what, count(*) AS rows,
	   count(*) FILTER (WHERE n IS DISTINCT FROM heap_n) AS differ
FROM (SELECT p.what, p.n,
			 CASE p.what
				 WHEN 'group' THEN (SELECT count(*) FROM pos_grouped
									WHERE d @@ 'everywhere <-> again' AND cat::text = p.k)
				 WHEN 'distinct' THEN (SELECT count(DISTINCT cat) FROM pos_grouped
									   WHERE d @@ 'common <-> again & !new5')
				 ELSE (SELECT count(*) FROM pos_grouped JOIN pos_dim
					   ON pos_grouped.cat = pos_dim.id
					   WHERE pos_grouped.d @@ 'w1:A | (common <-> again)'
					   AND pos_dim.name = p.k)
			 END AS heap_n
	  FROM pos_pushed p) x
GROUP BY what ORDER BY what;
   what   | rows | differ 
----------+------+--------
 distinct |    1 |      0
 group    |    7 |      0
 join     |    3 |      0
(3 rows)

RESET pg_lion.enable_count_pushdown;
RESET enable_bitmapscan;
RESET enable_indexscan;
-- Under an OR no recheck sees a leaf alone, so a leaf the sets only bound is
-- decided from the positions as the count walks it (LION_KN_POSFILTER): two
-- queries ORed, a query ORed with another column, the same with a GROUP BY,
-- and `@@ ANY (array)`, which is the OR of its elements' queries, each
-- counted by the pushdown and by the heap.  At the smallest
-- work_mem the union is read a window at a time, without its pins.
CREATE FUNCTION pos_or_count(sql text, pushed OUT bool, n OUT text)
LANGUAGE plpgsql AS $$
DECLARE
	l text;
BEGIN
	pushed := false;
	FOR l IN EXECUTE 'EXPLAIN (COSTS OFF) ' || sql LOOP
		pushed := pushed OR l LIKE '%Custom Scan (LionCount)%';
	END LOOP;
	EXECUTE 'SELECT string_agg(x::text, '','' ORDER BY x::text) FROM (' ||
		sql || ') x' INTO n;
END $$;
CREATE TEMP TABLE pos_or_shapes AS
SELECT format('SELECT count(*) FROM pos_grouped WHERE d @@ %L OR d @@ %L',
			  a.q, b.q) AS s
FROM pos_queries a, pos_queries b WHERE a.q::text < b.q::text
UNION ALL
SELECT format('SELECT count(*) FROM pos_grouped WHERE d @@ %L OR cat = 3', q)
FROM pos_queries
UNION ALL
SELECT format('SELECT cat, count(*) FROM pos_grouped WHERE d @@ %L OR d @@ %L '
			  'GROUP BY cat', a.q, b.q)
FROM pos_queries a, pos_queries b
WHERE a.q::text < b.q::text AND b.q IN ('common <-> again', 'w1:A')
UNION ALL
SELECT format('SELECT count(*) FROM pos_grouped WHERE d @@ ANY (%L::tsquery[])',
			  ARRAY[a.q, b.q])
FROM pos_queries a, pos_queries b WHERE a.q::text < b.q::text
UNION ALL
SELECT format('SELECT cat, count(*) FROM pos_grouped '
			  'WHERE d @@ ANY (%L::tsquery[]) GROUP BY cat', ARRAY[q, NULL, q])
FROM pos_queries;
-- core's GROUP BYs hash, over a scan whose @@ core charges far below its
-- time, and a small one can come out the cheaper (DESIGN.md §39, "A hashed
-- aggregate's own rate"); so can the sequential scan itself, its @@ an
-- operator call a row, against a count whose position filters are priced at
-- what decoding the positions takes (lion_cost_posfilter()), and a GROUP BY
-- of a row or two is a near tie with a sort of the index's own scan.
-- Hashing, sorts and sequential scans are off here, so that every shape is
-- counted
SET enable_hashagg = off;
SET enable_seqscan = off;
SET enable_sort = off;
CREATE TEMP TABLE pos_or_pushed AS
SELECT s, (pos_or_count(s)).* FROM pos_or_shapes;
SET work_mem = '64kB';
CREATE TEMP TABLE pos_or_narrow AS
SELECT s, (pos_or_count(s)).* FROM pos_or_shapes;
RESET work_mem;
RESET enable_hashagg;
RESET enable_seqscan;
RESET enable_sort;
SET pg_lion.enable_count_pushdown = off;
SET enable_bitmapscan = off;
SET enable_indexscan = off;
CREATE TEMP TABLE pos_or_heap AS
SELECT s, (pos_or_count(s)).* FROM pos_or_shapes;
RESET pg_lion.enable_count_pushdown;
RESET enable_bitmapscan;
RESET enable_indexscan;
-- every shape but those with a prefix lexeme is pushed down
SELECT count(*) AS shapes,
	   count(*) FILTER (WHERE p.pushed) AS pushed,
	   count(*) FILTER (WHERE NOT p.pushed AND p.s NOT LIKE '%:*%') AS not_pushed,
	   count(*) FILTER (WHERE p.n IS DISTINCT FROM h.n) AS differ,
	   count(*) FILTER (WHERE w.n IS DISTINCT FROM h.n) AS differ_narrow
FROM pos_or_pushed p JOIN pos_or_narrow w USING (s) JOIN pos_or_heap h USING (s);
 shapes | pushed | not_pushed | differ | differ_narrow 
--------+--------+------------+--------+---------------
    901 |    841 |          0 |      0 |             0
(1 row)

DO $$
DECLARE
	l text;
BEGIN
	FOR l IN EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF)
		SELECT count(*) FROM pos_grouped
		WHERE d @@ 'common <-> again' OR d @@ 'w1:A' LOOP
		IF l ~ '(Custom Scan|Heap TIDs Rechecked|Position|Removed)' THEN
			RAISE WARNING '%', regexp_replace(l, ' \(actual .*\)$', '');
		END IF;
	END LOOP;
END $$;
WARNING:  Custom Scan (LionCount)
WARNING:    Heap TIDs Rechecked: 0
WARNING:    Position Checks: 986
WARNING:    Rows Removed by Positions: 1
DROP FUNCTION pos_or_count(text);
DROP TABLE pos_grouped, pos_dim;
-- store_positions is "auto" by default (below), and read at build time only
-- and recorded on the meta page: ALTER INDEX changes nothing until a
-- REINDEX.  An explicit "on" (or true) on an index none of whose columns can
-- store positions is an ERROR.
CREATE TABLE pos_opt (id int, d tsvector) WITH (autovacuum_enabled = off);
INSERT INTO pos_opt
SELECT i, to_tsvector('simple', 'a b c ' || (i % 7)) FROM generate_series(1, 500) i;
CREATE INDEX pos_opt_d ON pos_opt USING lion (d) WITH (store_positions = off);
SELECT count(*) FROM lion_debug_key_positions('pos_opt_d', 'a');
ERROR:  index "pos_opt_d" does not store positions
CREATE INDEX pos_opt_id ON pos_opt USING lion (id) WITH (store_positions = true);
ERROR:  store_positions needs a key column whose operator class can store positions
CREATE INDEX pos_opt_id ON pos_opt USING lion (id) WITH (store_positions = on);
ERROR:  store_positions needs a key column whose operator class can store positions
ALTER INDEX pos_opt_d SET (store_positions = true);
INSERT INTO pos_opt VALUES (501, to_tsvector('simple', 'a b'));
SELECT count(*) FROM lion_debug_key_positions('pos_opt_d', 'a');
ERROR:  index "pos_opt_d" does not store positions
REINDEX INDEX pos_opt_d;
SELECT count(*) FROM lion_debug_key_positions('pos_opt_d', 'a');
 count 
-------
   501
(1 row)

ALTER INDEX pos_opt_d SET (store_positions = false);
INSERT INTO pos_opt VALUES (502, to_tsvector('simple', 'a b'));
SELECT count(*) FROM lion_debug_key_positions('pos_opt_d', 'a');
 count 
-------
   502
(1 row)

-- an OR of phrases is pushed down only while the index stores positions
VACUUM pos_opt;
SET enable_seqscan = off;
EXPLAIN (COSTS OFF)
SELECT count(*) FROM pos_opt WHERE d @@ 'a <-> b' OR d @@ 'b <-> c';
                                                  QUERY PLAN                                                   
---------------------------------------------------------------------------------------------------------------
 Custom Scan (LionCount)
   Lion Indexes: pos_opt_d, pos_opt_d ((d @@ '''a'' <-> ''b'''::tsquery) OR (d @@ '''b'' <-> ''c'''::tsquery))
(2 rows)

SELECT count(*) FROM pos_opt WHERE d @@ 'a <-> b' OR d @@ 'b <-> c';
 count 
-------
   502
(1 row)

REINDEX INDEX pos_opt_d;
SELECT count(*) FROM lion_debug_key_positions('pos_opt_d', 'a');
ERROR:  index "pos_opt_d" does not store positions
EXPLAIN (COSTS OFF)
SELECT count(*) FROM pos_opt WHERE d @@ 'a <-> b' OR d @@ 'b <-> c';
                                     QUERY PLAN                                      
-------------------------------------------------------------------------------------
 Aggregate
   ->  Index Scan using pos_opt_d on pos_opt
         Index Cond: (d @@ ANY ('{"''a'' <-> ''b''","''b'' <-> ''c''"}'::tsquery[]))
(3 rows)

SELECT count(*) FROM pos_opt WHERE d @@ 'a <-> b' OR d @@ 'b <-> c';
 count 
-------
   502
(1 row)

-- RESET is the default, "auto", which the next REINDEX stores them under
ALTER INDEX pos_opt_d RESET (store_positions);
SELECT count(*) FROM lion_debug_key_positions('pos_opt_d', 'a');
ERROR:  index "pos_opt_d" does not store positions
REINDEX INDEX pos_opt_d;
SELECT count(*) FROM lion_debug_key_positions('pos_opt_d', 'a');
 count 
-------
   502
(1 row)

EXPLAIN (COSTS OFF)
SELECT count(*) FROM pos_opt WHERE d @@ 'a <-> b' OR d @@ 'b <-> c';
                                                  QUERY PLAN                                                   
---------------------------------------------------------------------------------------------------------------
 Custom Scan (LionCount)
   Lion Indexes: pos_opt_d, pos_opt_d ((d @@ '''a'' <-> ''b'''::tsquery) OR (d @@ '''b'' <-> ''c'''::tsquery))
(2 rows)

SELECT count(*) FROM pos_opt WHERE d @@ 'a <-> b' OR d @@ 'b <-> c';
 count 
-------
   502
(1 row)

RESET enable_seqscan;
DROP TABLE pos_opt;
-- The default, store_positions = auto: positions for an index with a key
-- column whose opclass can store them (support function 5, tsvector_ops),
-- as GIN keeps them, and none, with no ERROR, for an index without one.
-- The meta page records what the build did: LION_META_POSITIONS is 2 in
-- order_flags, at +36 past the 24-byte page header (narrow.sql reads the
-- same word), and of an unlogged index's init fork, which ambuildempty
-- writes, as much as of its main fork.
CREATE EXTENSION IF NOT EXISTS pageinspect;
CREATE FUNCTION pos_meta(idx regclass, fork text) RETURNS text LANGUAGE sql AS $$
	SELECT CASE WHEN get_byte(p, 60) & 2 <> 0 THEN 'positions' ELSE 'none' END
	  FROM get_raw_page(idx::text, fork, 0) p
$$;
CREATE TABLE pos_def (id int, d tsvector) WITH (autovacuum_enabled = off);
INSERT INTO pos_def
SELECT i, setweight(to_tsvector('simple', 'w' || (i % 7)),
					CASE WHEN i % 5 = 0 THEN 'A' ELSE 'D' END::"char") ||
		  to_tsvector('simple', 'x' || (i % 11) || ' common')
  FROM generate_series(1, 7000) i;
CREATE INDEX pos_def_d ON pos_def USING lion (d);
CREATE INDEX pos_def_auto ON pos_def USING lion (d) WITH (store_positions = auto);
CREATE INDEX pos_def_on ON pos_def USING lion (d) WITH (store_positions = on);
CREATE INDEX pos_def_bare ON pos_def USING lion (d) WITH (store_positions);
CREATE INDEX pos_def_off ON pos_def USING lion (d) WITH (store_positions = off);
CREATE INDEX pos_def_false ON pos_def USING lion (d) WITH (store_positions = false);
CREATE INDEX pos_def_id ON pos_def USING lion (id);
CREATE INDEX pos_def_id_auto ON pos_def USING lion (id) WITH (store_positions = auto);
CREATE INDEX pos_def_id_off ON pos_def USING lion (id) WITH (store_positions = off);
CREATE INDEX pos_def_id_d ON pos_def USING lion (id, d);
CREATE INDEX pos_def_bad ON pos_def USING lion (d) WITH (store_positions = maybe);
ERROR:  invalid value for enum option "store_positions": maybe
SELECT c.relname, c.reloptions, pos_meta(c.oid, 'main') AS meta
  FROM pg_class c JOIN pg_index i ON i.indexrelid = c.oid
 WHERE i.indrelid = 'pos_def'::regclass ORDER BY 1;
     relname     |       reloptions        |   meta    
-----------------+-------------------------+-----------
 pos_def_auto    | {store_positions=auto}  | positions
 pos_def_bare    | {store_positions=true}  | positions
 pos_def_d       |                         | positions
 pos_def_false   | {store_positions=false} | none
 pos_def_id      |                         | none
 pos_def_id_auto | {store_positions=auto}  | none
 pos_def_id_d    |                         | positions
 pos_def_id_off  | {store_positions=off}   | none
 pos_def_off     | {store_positions=off}   | none
 pos_def_on      | {store_positions=on}    | positions
(10 rows)

SELECT count(*) FROM lion_debug_key_positions('pos_def_d', 'common');
 count 
-------
  7000
(1 row)

SELECT lion_index_verify(c.oid, true)
  FROM pg_class c JOIN pg_index i ON i.indexrelid = c.oid
 WHERE i.indrelid = 'pos_def'::regclass ORDER BY c.relname;
 lion_index_verify 
-------------------
 
 
 
 
 
 
 
 
 
 
(10 rows)

CREATE UNLOGGED TABLE pos_unl (id int, d tsvector);
CREATE INDEX pos_unl_d ON pos_unl USING lion (d);
CREATE INDEX pos_unl_d_off ON pos_unl USING lion (d) WITH (store_positions = off);
CREATE INDEX pos_unl_id ON pos_unl USING lion (id);
CREATE INDEX pos_unl_id_d ON pos_unl USING lion (id, d);
CREATE INDEX pos_unl_id_on ON pos_unl USING lion (id) WITH (store_positions = on);
ERROR:  store_positions needs a key column whose operator class can store positions
SELECT c.relname, pos_meta(c.oid, 'main') AS main, pos_meta(c.oid, 'init') AS init
  FROM pg_class c JOIN pg_index i ON i.indexrelid = c.oid
 WHERE i.indrelid = 'pos_unl'::regclass ORDER BY 1;
    relname    |   main    |   init    
---------------+-----------+-----------
 pos_unl_d     | positions | positions
 pos_unl_d_off | none      | none
 pos_unl_id    | none      | none
 pos_unl_id_d  | positions | positions
(4 rows)

DROP TABLE pos_unl;
-- BM25 needs nothing named: the default index ranks, one WITH
-- (store_positions = off) cannot
DROP INDEX pos_def_auto, pos_def_on, pos_def_bare, pos_def_false,
		   pos_def_id, pos_def_id_auto, pos_def_id_off, pos_def_id_d;
VACUUM (FREEZE, ANALYZE) pos_def;
SELECT count(*) AS ranked FROM lion_bm25('pos_def_d', 'w1 | x1', 10);
 ranked 
--------
     10
(1 row)

SELECT count(*) FROM lion_bm25('pos_def_off', 'w1 | x1', 10);
ERROR:  lion index "pos_def_off" does not store positions
-- With only the default index on the table, a phrase, a weight and a NOT
-- are answered from it without the heap: the bitmap heap scan rechecks
-- nothing, and the count takes its rows from the positions.  The index
-- WITH (store_positions = off), alone, rechecks them in the heap.
CREATE FUNCTION pos_def_explain(q tsquery) RETURNS SETOF text LANGUAGE plpgsql AS $$
DECLARE
	l text;
BEGIN
	SET LOCAL enable_seqscan = off;
	SET LOCAL enable_indexscan = off;
	FOR l IN EXECUTE format('EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) '
							'SELECT id FROM pos_def WHERE d @@ %L', q) LOOP
		IF l ~ '(Scan|Removed by Index Recheck)' THEN
			RETURN NEXT regexp_replace(l, ' \(actual .*\)$', '');
		END IF;
	END LOOP;
	SET LOCAL enable_bitmapscan = off;
	SET LOCAL enable_indexscan = on;
	FOR l IN EXECUTE format('EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) '
							'SELECT count(*) FROM pos_def WHERE d @@ %L', q) LOOP
		IF l ~ '(Custom Scan|Lion Indexes|Heap TIDs Rechecked|Position|Removed)' THEN
			RETURN NEXT regexp_replace(l, ' \(actual .*\)$', '');
		END IF;
	END LOOP;
END $$;
CREATE TEMP TABLE pos_def_q (q tsquery);
INSERT INTO pos_def_q VALUES ('w1 <-> x1'), ('x1 <-> w1'), ('w1:A'), ('w1 & !x1');
DROP INDEX pos_def_off;
SELECT q, pos_def_explain(q) FROM pos_def_q;
       q       |                        pos_def_explain                        
---------------+---------------------------------------------------------------
 'w1' <-> 'x1' | Bitmap Heap Scan on pos_def
 'w1' <-> 'x1' |   ->  Bitmap Index Scan on pos_def_d
 'w1' <-> 'x1' | Custom Scan (LionCount)
 'w1' <-> 'x1' |   Lion Indexes: pos_def_d (d @@ '''w1'' <-> ''x1'''::tsquery)
 'w1' <-> 'x1' |   Heap TIDs Rechecked: 0
 'w1' <-> 'x1' |   Position Checks: 91
 'w1' <-> 'x1' |   Rows Removed by Positions: 0
 'x1' <-> 'w1' | Index Scan using pos_def_d on pos_def
 'x1' <-> 'w1' | Custom Scan (LionCount)
 'x1' <-> 'w1' |   Lion Indexes: pos_def_d (d @@ '''x1'' <-> ''w1'''::tsquery)
 'x1' <-> 'w1' |   Heap TIDs Rechecked: 0
 'x1' <-> 'w1' |   Position Checks: 91
 'x1' <-> 'w1' |   Rows Removed by Positions: 91
 'w1':A        | Bitmap Heap Scan on pos_def
 'w1':A        |   ->  Bitmap Index Scan on pos_def_d
 'w1':A        | Custom Scan (LionCount)
 'w1':A        |   Lion Indexes: pos_def_d (d @@ '''w1'':A'::tsquery)
 'w1':A        |   Heap TIDs Rechecked: 0
 'w1':A        |   Position Checks: 1000
 'w1':A        |   Rows Removed by Positions: 800
 'w1' & !'x1'  | Bitmap Heap Scan on pos_def
 'w1' & !'x1'  |   ->  Bitmap Index Scan on pos_def_d
 'w1' & !'x1'  | Custom Scan (LionCount)
 'w1' & !'x1'  |   Lion Indexes: pos_def_d (d @@ '''w1'' & !''x1'''::tsquery)
 'w1' & !'x1'  |   Heap TIDs Rechecked: 0
 'w1' & !'x1'  |   Position Checks: 1000
 'w1' & !'x1'  |   Rows Removed by Positions: 91
(27 rows)

SET enable_seqscan = off;
CREATE TEMP TABLE pos_def_n AS
SELECT q::text, (SELECT count(*) FROM pos_def WHERE d @@ q) AS by_positions FROM pos_def_q;
DROP INDEX pos_def_d;
CREATE INDEX pos_def_off ON pos_def USING lion (d) WITH (store_positions = off);
SELECT q, pos_def_explain(q) FROM pos_def_q;
       q       |                         pos_def_explain                         
---------------+-----------------------------------------------------------------
 'w1' <-> 'x1' | Bitmap Heap Scan on pos_def
 'w1' <-> 'x1' |   ->  Bitmap Index Scan on pos_def_off
 'w1' <-> 'x1' | Custom Scan (LionCount)
 'w1' <-> 'x1' |   Lion Indexes: pos_def_off (d @@ '''w1'' <-> ''x1'''::tsquery)
 'w1' <-> 'x1' |   Heap TIDs Rechecked: 91
 'x1' <-> 'w1' | Index Scan using pos_def_off on pos_def
 'x1' <-> 'w1' |   Rows Removed by Index Recheck: 91
 'x1' <-> 'w1' | Custom Scan (LionCount)
 'x1' <-> 'w1' |   Lion Indexes: pos_def_off (d @@ '''x1'' <-> ''w1'''::tsquery)
 'x1' <-> 'w1' |   Heap TIDs Rechecked: 91
 'x1' <-> 'w1' |   Rows Removed by Recheck: 91
 'w1':A        | Bitmap Heap Scan on pos_def
 'w1':A        |   Rows Removed by Index Recheck: 800
 'w1':A        |   ->  Bitmap Index Scan on pos_def_off
 'w1':A        | Custom Scan (LionCount)
 'w1':A        |   Lion Indexes: pos_def_off (d @@ '''w1'':A'::tsquery)
 'w1':A        |   Heap TIDs Rechecked: 1000
 'w1':A        |   Rows Removed by Recheck: 800
 'w1' & !'x1'  | Bitmap Heap Scan on pos_def
 'w1' & !'x1'  |   Rows Removed by Index Recheck: 91
 'w1' & !'x1'  |   ->  Bitmap Index Scan on pos_def_off
 'w1' & !'x1'  | Custom Scan (LionCount)
 'w1' & !'x1'  |   Lion Indexes: pos_def_off (d @@ '''w1'' & !''x1'''::tsquery)
 'w1' & !'x1'  |   Heap TIDs Rechecked: 1000
 'w1' & !'x1'  |   Rows Removed by Recheck: 91
(25 rows)

-- the same rows either way, and a sequential scan's
SELECT q, n.by_positions,
	   (SELECT count(*) FROM pos_def WHERE d @@ q::tsquery) AS by_recheck,
	   (SELECT count(*) FROM pos_def WHERE (d::text || '')::tsvector @@ q::tsquery) AS heap
  FROM pos_def_n n ORDER BY 1;
       q       | by_positions | by_recheck | heap 
---------------+--------------+------------+------
 'w1' & !'x1'  |          909 |        909 |  909
 'w1' <-> 'x1' |           91 |         91 |   91
 'w1':A        |          200 |        200 |  200
 'x1' <-> 'w1' |            0 |          0 |    0
(4 rows)

RESET enable_seqscan;
DROP FUNCTION pos_def_explain(tsquery);
DROP FUNCTION pos_meta(regclass, text);
DROP TABLE pos_def;
DROP TABLE pos_docs CASCADE;
DROP FUNCTION lion_debug_key_positions(regclass, text);