-- The dictionary of a string leaf is one compressed blob that every scan reading the leaf inflates and indexes whole. A segment never
-- changes, so the backend keeps the parsed dictionary for the next query (col/dictcache.c): the second scan parses none, a rewritten
-- segment has its own, and the answers do not depend on it.
CREATE EXTENSION pg_sift;
CREATE EXTENSION pg_sift_test;
SET sift.test_col_legacy_options = on;
CREATE TABLE dc (id int, c0 text, c1 text, c2 text, c3 text, c4 text, c5 text);
INSERT INTO dc SELECT g, v, v, v, v, v, v FROM (SELECT g, 'label ' || (g % 500) AS v FROM generate_series(1, 20000) g) s;
CREATE INDEX dc_idx ON dc USING sift_col (c0, c1, c2, c3, c4, c5);
NOTICE:  sift_col index "dc_idx": 1 advice (track_changes)
DETAIL:  advice: track_changes = the table has no one-column primary key
HINT:  Reasons, costs and how to turn each off: SELECT * FROM sift.index_choices('dc_idx');
SELECT count(*) > 0 AS has_segment FROM sift.col_segments('dc_idx'::regclass) WHERE kind = 'immutable';
 has_segment 
-------------
 t
(1 row)

-- a cache that holds five of the six dictionaries (the backend starts with none): the least recently used goes, the answers do not change
SELECT count(*) > 0 AS reset FROM sift._test_col_dict_cache(true);
 reset 
-------
 t
(1 row)

SELECT rows FROM sift._test_col_counters('dc_idx', '0:eq:label 7');
 rows 
------
   40
(1 row)

SELECT bytes AS one FROM sift._test_col_dict_cache(true) \gset
SELECT set_config('sift.test_limits', 'dict_cache_kb=' || ((:one * 5.5 + 1023) / 1024)::int, false) IS NOT NULL AS sized;
 sized 
-------
 t
(1 row)

SELECT sum(rows) AS rows FROM generate_series(1, 5) l, LATERAL sift._test_col_counters('dc_idx', l || ':eq:label 7');
 rows 
------
  200
(1 row)

SELECT loads = 5 AS parsed, bytes <= :one * 5.5 AS within_the_limit FROM sift._test_col_dict_cache(true);
 parsed | within_the_limit 
--------+------------------
 t      | t
(1 row)

SELECT rows FROM sift._test_col_counters('dc_idx', '5:eq:label 7');
 rows 
------
   40
(1 row)

SELECT rows FROM sift._test_col_counters('dc_idx', '0:eq:label 7');
 rows 
------
   40
(1 row)

SELECT rows FROM sift._test_col_counters('dc_idx', '5:eq:label 7');
 rows 
------
   40
(1 row)

SELECT loads = 1 AS first_parsed_again, hits = 2 AS the_others_found FROM sift._test_col_dict_cache(true);
 first_parsed_again | the_others_found 
--------------------+------------------
 t                  | t
(1 row)

RESET sift.test_limits;
-- one dictionary may take up to a half of the cache: a third of it is kept, one over a half is parsed for every scan, the answers do not change
REINDEX INDEX dc_idx;
NOTICE:  sift_col index "dc_idx": 1 advice (track_changes)
DETAIL:  advice: track_changes = the table has no one-column primary key
HINT:  Reasons, costs and how to turn each off: SELECT * FROM sift.index_choices('dc_idx');
SELECT set_config('sift.test_limits', 'dict_cache_kb=' || ((:one * 3 + 1023) / 1024)::int, false) IS NOT NULL AS sized;
 sized 
-------
 t
(1 row)

SELECT count(*) > 0 AS reset FROM sift._test_col_dict_cache(true);
 reset 
-------
 t
(1 row)

SELECT rows FROM sift._test_col_counters('dc_idx', '3:eq:label 7');
 rows 
------
   40
(1 row)

SELECT rows FROM sift._test_col_counters('dc_idx', '3:eq:label 7');
 rows 
------
   40
(1 row)

SELECT loads = 1 AS parsed_once, hits = 1 AS found_again FROM sift._test_col_dict_cache(true);
 parsed_once | found_again 
-------------+-------------
 t           | t
(1 row)

SELECT set_config('sift.test_limits', 'dict_cache_kb=' || ((:one * 1.5 + 1023) / 1024)::int, false) IS NOT NULL AS sized;
 sized 
-------
 t
(1 row)

SELECT rows FROM sift._test_col_counters('dc_idx', '4:eq:label 7');
 rows 
------
   40
(1 row)

SELECT rows FROM sift._test_col_counters('dc_idx', '4:eq:label 7');
 rows 
------
   40
(1 row)

SELECT loads = 2 AS parsed_each_time, hits = 0 AS none_found FROM sift._test_col_dict_cache(true);
 parsed_each_time | none_found 
------------------+------------
 t                | t
(1 row)

RESET sift.test_limits;
-- the first scan of a segment parses a dictionary, the second finds it
REINDEX INDEX dc_idx;
NOTICE:  sift_col index "dc_idx": 1 advice (track_changes)
DETAIL:  advice: track_changes = the table has no one-column primary key
HINT:  Reasons, costs and how to turn each off: SELECT * FROM sift.index_choices('dc_idx');
SELECT rows FROM sift._test_col_counters('dc_idx', '0:eq:label 7');
 rows 
------
   40
(1 row)

SELECT loads > 0 AS parsed, hits = 0 AS none_found FROM sift._test_col_dict_cache(true);
 parsed | none_found 
--------+------------
 t      | t
(1 row)

SELECT rows FROM sift._test_col_counters('dc_idx', '0:eq:label 7');
 rows 
------
   40
(1 row)

SELECT loads = 0 AS none_parsed, hits > 0 AS found FROM sift._test_col_dict_cache(true);
 none_parsed | found 
-------------+-------
 t           | t
(1 row)

-- another leaf has its own
SELECT rows FROM sift._test_col_counters('dc_idx', '1:eq:label 7');
 rows 
------
   40
(1 row)

SELECT loads > 0 AS parsed, hits = 0 AS none_found FROM sift._test_col_dict_cache(true);
 parsed | none_found 
--------+------------
 t      | t
(1 row)

-- a segment written again (REINDEX) is another segment: its dictionary is parsed, the answers are the same
REINDEX INDEX dc_idx;
NOTICE:  sift_col index "dc_idx": 1 advice (track_changes)
DETAIL:  advice: track_changes = the table has no one-column primary key
HINT:  Reasons, costs and how to turn each off: SELECT * FROM sift.index_choices('dc_idx');
SELECT rows FROM sift._test_col_counters('dc_idx', '0:eq:label 7');
 rows 
------
   40
(1 row)

SELECT loads > 0 AS parsed FROM sift._test_col_dict_cache(true);
 parsed 
--------
 t
(1 row)

SELECT rows FROM sift._test_col_counters('dc_idx', '0:eq:label 7');
 rows 
------
   40
(1 row)

SELECT loads = 0 AS none_parsed, hits > 0 AS found FROM sift._test_col_dict_cache(true);
 none_parsed | found 
-------------+-------
 t           | t
(1 row)

-- a cache too small to keep a dictionary parses it every time, with the same answers
SET sift.test_limits = 'dict_cache_kb=1';
SELECT rows FROM sift._test_col_counters('dc_idx', '2:eq:label 7');
 rows 
------
   40
(1 row)

SELECT rows FROM sift._test_col_counters('dc_idx', '2:eq:label 7');
 rows 
------
   40
(1 row)

SELECT loads = 2 AS parsed_each_time, hits = 0 AS none_found FROM sift._test_col_dict_cache(true);
 parsed_each_time | none_found 
------------------+------------
 t                | t
(1 row)

RESET sift.test_limits;
DROP TABLE dc;
-- a dictionary of thousands of strings is searched in the stream, and parsed and kept once the backend has tested it often; when the cache cannot hold it that
-- parse is not repeated for every test (it costs thousands of tests of the stream), and the answers do not change
CREATE TABLE dl (id int, s text);
INSERT INTO dl SELECT g, 'key ' || g FROM generate_series(1, 20000) g;
CREATE INDEX dl_idx ON dl USING sift_col (id sift.col_int4_ops, s sift.col_text_ops) WITH (point_columns = 's');
NOTICE:  sift_col index "dl_idx": 1 advice (track_changes)
DETAIL:  advice: track_changes = the table has no one-column primary key
HINT:  Reasons, costs and how to turn each off: SELECT * FROM sift.index_choices('dl_idx');
SET sift.test_limits = 'dict_cache_kb=1';
SELECT count(*) > 0 AS reset FROM sift._test_col_dict_cache(true);
 reset 
-------
 t
(1 row)

SELECT sum(rows) AS rows, count(*) AS tests FROM generate_series(1, 100) l, LATERAL sift._test_col_counters('dl_idx', '1:eq:key ' || l * 7);
 rows | tests 
------+-------
  100 |   100
(1 row)

SELECT loads <= 2 AS parsed_at_most_twice, hits = 0 AS none_found FROM sift._test_col_dict_cache(true);
 parsed_at_most_twice | none_found 
----------------------+------------
 t                    | t
(1 row)

RESET sift.test_limits;
DROP TABLE dl;
-- the lengths of the lists of a unit are decoded once as well (integer elements: no dictionary is involved): the second scan finds them, the answers are the same
CREATE TABLE lc (id int, parent_id int, parts jsonb) WITH (autovacuum_enabled = false);
INSERT INTO lc SELECT g, 1 + g % 50, jsonb_build_array(jsonb_build_object('qty', g % 7, 'price', g % 11), jsonb_build_object('qty', (g + 1) % 7, 'price', (g + 3) % 11)) FROM generate_series(1, 20000) g;
CREATE INDEX lc_idx ON lc USING sift_col (id, parent_id, parts) WITH (paths = 'parts[]{qty int4, price int4}');
NOTICE:  sift_col index "lc_idx": 1 advice (track_changes)
DETAIL:  advice: track_changes = the table has no one-column primary key
HINT:  Reasons, costs and how to turn each off: SELECT * FROM sift.index_choices('lc_idx');
VACUUM (FREEZE, ANALYZE) lc;
SET max_parallel_workers_per_gather = 0;
SET seq_page_cost = 1000; SET random_page_cost = 1000;
\set q 'SELECT e.qty, count(*) AS n, sum(e.price) AS s FROM lc f CROSS JOIN LATERAL jsonb_to_recordset(f.parts) e(qty int, price int) WHERE f.parent_id IN (3, 4) GROUP BY 1'
CREATE TEMP TABLE lc_first AS :q;
SELECT loads > 0 AS lengths_decoded, hits = 0 AS none_found FROM sift._test_col_dict_cache(true);
 lengths_decoded | none_found 
-----------------+------------
 t               | t
(1 row)

CREATE TEMP TABLE lc_second AS :q;
SELECT loads = 0 AS none_decoded, hits > 0 AS found FROM sift._test_col_dict_cache(true);
 none_decoded | found 
--------------+-------
 t            | t
(1 row)

SET sift.enable_agg = off;
CREATE TEMP TABLE lc_off AS :q;
RESET sift.enable_agg;
SELECT (SELECT count(*) FROM lc_first) AS groups, (SELECT count(*) FROM ((TABLE lc_second EXCEPT TABLE lc_off) UNION ALL (TABLE lc_off EXCEPT TABLE lc_second)) d) AS differences;
 groups | differences 
--------+-------------
      7 |           0
(1 row)

-- a cache too small to keep them (of a segment written again: the older ones are still found) decodes them every time, with the same answers
SET sift.test_limits = 'dict_cache_kb=1';
REINDEX INDEX lc_idx;
NOTICE:  sift_col index "lc_idx": 1 advice (track_changes)
DETAIL:  advice: track_changes = the table has no one-column primary key
HINT:  Reasons, costs and how to turn each off: SELECT * FROM sift.index_choices('lc_idx');
SELECT count(*) > 0 AS reset FROM sift._test_col_dict_cache(true);
 reset 
-------
 t
(1 row)

CREATE TEMP TABLE lc_small AS :q;
CREATE TEMP TABLE lc_small2 AS :q;
SELECT hits = 0 AS none_found FROM sift._test_col_dict_cache(true);
 none_found 
------------
 t
(1 row)

SELECT (SELECT count(*) FROM ((TABLE lc_small2 EXCEPT TABLE lc_off) UNION ALL (TABLE lc_off EXCEPT TABLE lc_small2)) d) AS differences;
 differences 
-------------
           0
(1 row)

RESET sift.test_limits;
DROP TABLE lc;
DROP EXTENSION pg_sift_test;
DROP EXTENSION pg_sift;