-- The blob of a dense term (8 kB or more) is kept by the backend under the segment's relation file and generation: the second
-- query reads none of it again, and no query is ever answered with the blob of another segment, of another index of the same
-- layout, or of an index that was rebuilt. The budget (the limit segment_cache_kb) only decides what is kept: a backend that keeps
-- nothing and one that keeps less than its working set (blobs evicted, others in use) answer alike.
CREATE EXTENSION pg_sift;
CREATE EXTENSION pg_sift_test;
CREATE FUNCTION body(g int) RETURNS text LANGUAGE sql IMMUTABLE AS
$$ SELECT 'dim_a' || g % 6 || ' dim_a' || (g + 1) % 6 || ' dim_a' || (g + 2) % 6 || ' w' || g % 50 $$;
CREATE TABLE docs_a (id int, body text, pad text);
CREATE TABLE docs_b (id int, body text, pad text);
INSERT INTO docs_a SELECT g, body(g), repeat('p', 700) FROM generate_series(1, 40960) g;
INSERT INTO docs_b SELECT 1000000 + g, body(g), repeat('p', 700) FROM generate_series(1, 40960) g;
CREATE INDEX docs_a_idx ON docs_a USING sift (body);
CREATE INDEX docs_b_idx ON docs_b USING sift (body);
-- the 25 best of a table by score, then by id up or down
CREATE FUNCTION top(tbl regclass, q text, up bool) RETURNS text LANGUAGE plpgsql AS $$
DECLARE r text;
BEGIN
	EXECUTE format('SELECT string_agg(id::text, '','' ORDER BY n) FROM (SELECT row_number() OVER () AS n, id FROM
		(SELECT id FROM %s WHERE body @@> $1 ORDER BY sift.score(ctid) DESC, CASE WHEN $2 THEN id ELSE -id END LIMIT 25) x) y', tbl) INTO r USING q, up;
	RETURN r;
END $$;
-- the bytes of blobs copied out of the index since the last reset, and the blobs found in the cache instead
CREATE FUNCTION copied() RETURNS bigint LANGUAGE sql AS $$ SELECT value FROM sift._test_read_stats() WHERE key = 'blob_bytes' $$;
CREATE FUNCTION found() RETURNS bigint LANGUAGE sql AS $$ SELECT value FROM sift._test_read_stats() WHERE key = 'blob_hits' $$;
CREATE TEMP TABLE qs (q text);
INSERT INTO qs VALUES ('dim_a0'), ('dim_a1 OR dim_a2'), ('dim_a3 dim_a4'), ('dim_a5 OR dim_a1'), ('dim_a0 OR dim_a1 OR dim_a2 OR dim_a3 OR dim_a4 OR dim_a5');
-- nothing kept: the reference (every pass copies all)
SET sift.test_limits = 'segment_cache_kb=0';
CREATE TEMP TABLE ref AS SELECT t, q, up, top(t, q, up) AS a FROM qs, unnest(ARRAY['docs_a', 'docs_b']::regclass[]) t, (VALUES (true), (false)) u (up);
SELECT found() AS found_nothing;
 found_nothing 
---------------
             0
(1 row)

SELECT copied() AS all_bytes \gset
RESET sift.test_limits;
-- the first pass finds what its own queries repeat, the second one copies only the blobs under 8 kB, which are not kept
SELECT count(*) > 0 AS reset FROM sift._test_read_stats(true);
 reset 
-------
 t
(1 row)

SELECT t, q, up FROM ref WHERE top(t, q, up) IS DISTINCT FROM a;
 t | q | up 
---+---+----
(0 rows)

SELECT copied() AS first_pass \gset
SELECT :first_pass < :all_bytes AS first_pass_copies_less, found() > 0 AS first_pass_found;
 first_pass_copies_less | first_pass_found 
------------------------+------------------
 t                      | t
(1 row)

SELECT count(*) > 0 AS reset FROM sift._test_read_stats(true);
 reset 
-------
 t
(1 row)

SELECT t, q, up FROM ref WHERE top(t, q, up) IS DISTINCT FROM a;
 t | q | up 
---+---+----
(0 rows)

SELECT copied() AS second_pass \gset
SELECT :second_pass < :first_pass AS second_pass_copies_less, found() > 0 AS found;
 second_pass_copies_less | found 
-------------------------+-------
 t                       | t
(1 row)

-- two indexes of one layout (same generation, offsets and sizes), each with its own rows
SELECT t, up, a FROM ref WHERE q = 'dim_a0' ORDER BY t, up;
   t    | up |                                                                                                    a                                                                                                    
--------+----+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 docs_a | f  | 40960,40956,40955,40954,40950,40949,40948,40944,40943,40942,40938,40937,40936,40932,40931,40930,40926,40925,40924,40920,40919,40918,40914,40913,40912
 docs_a | t  | 4,5,6,10,11,12,16,17,18,22,23,24,28,29,30,34,35,36,40,41,42,46,47,48,52
 docs_b | f  | 1040960,1040956,1040955,1040954,1040950,1040949,1040948,1040944,1040943,1040942,1040938,1040937,1040936,1040932,1040931,1040930,1040926,1040925,1040924,1040920,1040919,1040918,1040914,1040913,1040912
 docs_b | t  | 1000004,1000005,1000006,1000010,1000011,1000012,1000016,1000017,1000018,1000022,1000023,1000024,1000028,1000029,1000030,1000034,1000035,1000036,1000040,1000041,1000042,1000046,1000047,1000048,1000052
(4 rows)

-- a rebuilt index has a file of its own: the first pass copies again, with the same answers
REINDEX INDEX docs_a_idx;
SELECT count(*) > 0 AS reset FROM sift._test_read_stats(true);
 reset 
-------
 t
(1 row)

SELECT t, q, up FROM ref WHERE t = 'docs_a'::regclass AND top(t, q, up) IS DISTINCT FROM a;
 t | q | up 
---+---+----
(0 rows)

SELECT copied() AS rebuilt_first \gset
SELECT count(*) > 0 AS reset FROM sift._test_read_stats(true);
 reset 
-------
 t
(1 row)

SELECT t, q, up FROM ref WHERE t = 'docs_a'::regclass AND top(t, q, up) IS DISTINCT FROM a;
 t | q | up 
---+---+----
(0 rows)

SELECT :rebuilt_first > copied() AS copies_again;
 copies_again 
--------------
 t
(1 row)

-- a second segment of the index (same layout, another generation) is read from its own blobs
INSERT INTO docs_a SELECT 2000000 + g, body(g), repeat('p', 700) FROM generate_series(1, 40960) g;
SELECT sift.promote('docs_a_idx') > 0 AS promoted;
 promoted 
----------
 t
(1 row)

INSERT INTO docs_a SELECT 3000000 + g, body(g), repeat('p', 700) FROM generate_series(1, 40960) g;
SELECT sift.promote('docs_a_idx') > 0 AS promoted;
 promoted 
----------
 t
(1 row)

SELECT gen, kind, ndocs, sb_lo, sb_hi, postings_bytes FROM sift.segments('docs_a_idx'::regclass) ORDER BY gen;
 gen |   kind    | ndocs | sb_lo | sb_hi | postings_bytes 
-----+-----------+-------+-------+-------+----------------
   1 | immutable | 40960 |     0 |    15 |         262523
   4 | immutable | 40960 |    16 |    31 |         262601
   5 | buffer    |     0 |     0 |     0 |               
   6 | immutable | 40960 |    32 |    47 |         262601
(4 rows)

SELECT q, top('docs_a', q, false) AS newest_first FROM qs WHERE q IN ('dim_a0', 'dim_a1 OR dim_a2');
        q         |                                                                                              newest_first                                                                                               
------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 dim_a0           | 3040960,3040956,3040955,3040954,3040950,3040949,3040948,3040944,3040943,3040942,3040938,3040937,3040936,3040932,3040931,3040930,3040926,3040925,3040924,3040920,3040919,3040918,3040914,3040913,3040912
 dim_a1 OR dim_a2 | 3040957,3040956,3040951,3040950,3040945,3040944,3040939,3040938,3040933,3040932,3040927,3040926,3040921,3040920,3040915,3040914,3040909,3040908,3040903,3040902,3040897,3040896,3040891,3040890,3040885
(2 rows)

CREATE TEMP TABLE ref_a AS SELECT q, up, top('docs_a', q, up) AS a FROM qs, (VALUES (true), (false)) u (up);
-- and a rewritten one (the merge, then VACUUM)
SELECT sift.merge('docs_a_idx', true) >= 0 AS merged;
 merged 
--------
 t
(1 row)

SELECT q, up FROM ref_a WHERE top('docs_a', q, up) IS DISTINCT FROM a;
 q | up 
---+----
(0 rows)

DELETE FROM docs_a WHERE id % 3 = 0;
VACUUM docs_a;
SELECT q, up, top('docs_a', q, up) AS vacuumed FROM ref_a WHERE q = 'dim_a0' ORDER BY up;
   q    | up |                                                                                                vacuumed                                                                                                 
--------+----+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 dim_a0 | f  | 3040960,3040955,3040954,3040949,3040948,3040943,3040942,3040937,3040936,3040931,3040930,3040925,3040924,3040919,3040918,3040913,3040912,3040907,3040906,3040901,3040900,3040895,3040894,3040889,3040888
 dim_a0 | t  | 4,5,10,11,16,17,22,23,28,29,34,35,40,41,46,47,52,53,58,59,64,65,70,71,76
(2 rows)

SELECT count(*) AS deleted_found FROM docs_a WHERE body @@> 'dim_a0' AND id % 3 = 0;
 deleted_found 
---------------
             0
(1 row)

-- a budget below the working set (the blobs of the six terms are 9 kB each and are kept up to a quarter of the budget, so 4 fit; the shared blob
-- of 35 kB is not kept): blobs go out as others come in, none that a cursor holds (the query of the six terms finds the budget full of its own
-- and copies the others for itself), and the answers are those of the backend that keeps nothing
SET sift.test_limits = 'segment_cache_kb=40';
SELECT count(*) > 0 AS reset FROM sift._test_read_stats(true);
 reset 
-------
 t
(1 row)

SELECT t, q, up FROM ref WHERE top(t, q, up) IS DISTINCT FROM a AND t = 'docs_b'::regclass;
 t | q | up 
---+---+----
(0 rows)

SELECT t, q, up FROM ref WHERE top(t, q, up) IS DISTINCT FROM a AND t = 'docs_b'::regclass;
 t | q | up 
---+---+----
(0 rows)

SELECT max(value) FILTER (WHERE key = 'blob_hits') > 0 AS found, max(value) FILTER (WHERE key = 'segcache_evicted') > 0 AS evicted,
	   max(value) FILTER (WHERE key = 'segcache_refused') > 0 AS refused, max(value) FILTER (WHERE key = 'segcache_bytes') <= 40 * 1024 AS kept_within_budget
FROM sift._test_read_stats();
 found | evicted | refused | kept_within_budget 
-------+---------+---------+--------------------
 t     | t       | t       | t
(1 row)

RESET sift.test_limits;
DROP TABLE docs_a;
DROP TABLE docs_b;
DROP FUNCTION copied();
DROP FUNCTION found();
DROP FUNCTION top(regclass, text, bool);
DROP FUNCTION body(int);
DROP EXTENSION pg_sift_test;
DROP EXTENSION pg_sift;