-- The write buffer of a BM25 index in format 6 (SIFT_WBUF_FORMAT_WIDE) has a bloom of 8,192 bits for each full page, filled when the page is full, where format 0 has 256 bits for
-- the page, which a page of 120 terms a document fills: a term in no document of the buffer then reads every page. The rows are those of the buffer of format 0, of a buffer being appended to
-- (its pages have the small bloom), of the segments after the promotion, and of the heap; a buffer keeps the format it was opened with, and VACUUM, verify and deletes read both.
CREATE EXTENSION pg_sift;
CREATE EXTENSION pg_sift_test;
SET client_min_messages = warning;
SET sift.async_maintenance = off;
SET max_parallel_workers_per_gather = 0;
-- 120 terms in a document, 'common' in every document, the others spread so that a page of 6 documents holds 700 different ones
CREATE FUNCTION ww_doc(i int) RETURNS text LANGUAGE sql IMMUTABLE AS
$$ SELECT 'common ' || (SELECT string_agg('t' || ((i * 7 + k * 13) % 4001), ' ') FROM generate_series(1, 120) k) || ' ' || (ARRAY['red', 'green', 'blue'])[1 + i % 3] $$;
CREATE TABLE ww (id int, body text) WITH (autovacuum_enabled = false);
CREATE INDEX ww_body ON ww USING sift (body);
INSERT INTO ww SELECT i, ww_doc(i) FROM generate_series(1, 1200) i;
SET sift.test_wbuf_narrow = on;
CREATE TABLE wn (id int, body text) WITH (autovacuum_enabled = false);
CREATE INDEX wn_body ON wn USING sift (body);
INSERT INTO wn SELECT i, ww_doc(i) FROM generate_series(1, 1200) i;
RESET sift.test_wbuf_narrow;
-- the format of the buffer each opened with
SELECT 'ww' AS tbl, format, ndocs = 1200 AS all_in_the_buffer FROM sift._test_wbuf_reads('ww_body') UNION ALL SELECT 'wn', format, ndocs = 1200 FROM sift._test_wbuf_reads('wn_body') ORDER BY 1;
 tbl | format | all_in_the_buffer 
-----+--------+-------------------
 wn  |      0 | t
 ww  |      6 | t
(2 rows)

-- a term in no document: the wide bloom reads the pages being appended to and a few false ones, the small one nearly all
CREATE FUNCTION ww_walk(tbl text, q text) RETURNS bigint LANGUAGE plpgsql AS $$
DECLARE w0 bigint; w1 bigint;
BEGIN
	EXECUTE format('SELECT walked FROM sift._test_wbuf_reads(%L)', tbl || '_body') INTO w0;
	EXECUTE format('SELECT count(*) FROM %I WHERE body @@> %L', tbl, q);
	EXECUTE format('SELECT walked FROM sift._test_wbuf_reads(%L)', tbl || '_body') INTO w1;
	RETURN w1 - w0;
END $$;
SELECT ww_walk('ww', 'qqabsent') * 5 < ndocs AS wide_skips_pages, ww_walk('wn', 'qqabsent') * 10 > ndocs * 9 AS small_does_not, ww_walk('ww', 'common') = ndocs AS common_reads_all
FROM sift._test_wbuf_reads('ww_body');
 wide_skips_pages | small_does_not | common_reads_all 
------------------+----------------+------------------
 t                | t              | t
(1 row)

-- the same rows from the buffer of either format, the segments after promotion and the heap
CREATE FUNCTION ww_rows(tbl text, q text) RETURNS text LANGUAGE plpgsql AS $$
DECLARE r text;
BEGIN
	EXECUTE format('SELECT count(*) || '' '' || coalesce(sum(id), 0) FROM %I WHERE body @@> %L', tbl, q) INTO r;
	RETURN r;
END $$;
CREATE TABLE ww_q (n int, q text);
INSERT INTO ww_q VALUES (1, 't7'), (2, 'red'), (3, 'red blue'), (4, 'red OR green'), (5, 'common NOT red'), (6, 't7 t20 t33'), (7, 'qqabsent'), (8, 't1500 OR t3999'), (9, '"t7 t20"'), (10, 't1*');
SELECT n, q, ww_rows('ww', q) AS wide, ww_rows('ww', q) = ww_rows('wn', q) AS same_as_narrow FROM ww_q ORDER BY n;
 n  |       q        |    wide     | same_as_narrow 
----+----------------+-------------+----------------
  1 | t7             | 34 25364    | t
  2 | red            | 400 240600  | t
  3 | red blue       | 0 0         | t
  4 | red OR green   | 800 480400  | t
  5 | common NOT red | 800 480000  | t
  6 | t7 t20 t33     | 34 25364    | t
  7 | qqabsent       | 0 0         | t
  8 | t1500 OR t3999 | 74 46018    | t
  9 | "t7 t20"       | 34 25364    | t
 10 | t1*            | 1062 637219 | t
(10 rows)

-- appended to, updated, deleted
INSERT INTO ww SELECT i, ww_doc(i) FROM generate_series(1201, 1500) i;
INSERT INTO wn SELECT i, ww_doc(i) FROM generate_series(1201, 1500) i;
UPDATE ww SET body = ww_doc(id + 1) WHERE id % 7 = 0;
UPDATE wn SET body = ww_doc(id + 1) WHERE id % 7 = 0;
DELETE FROM ww WHERE id % 11 = 0;
DELETE FROM wn WHERE id % 11 = 0;
SELECT n, ww_rows('ww', q) = ww_rows('wn', q) AS same_as_narrow, ww_rows('ww', q) AS rows FROM ww_q ORDER BY n;
 n  | same_as_narrow |    rows     
----+----------------+-------------
  1 | t              | 29 21243
  2 | t              | 454 340559
  3 | t              | 0 0
  4 | t              | 908 680683
  5 | t              | 910 682715
  6 | t              | 29 21243
  7 | t              | 0 0
  8 | t              | 81 58268
  9 | t              | 29 21243
 10 | t              | 1179 859850
(10 rows)

VACUUM ww, wn;
SELECT sift.verify('ww_body'), sift.verify('wn_body');
 verify | verify 
--------+--------
        | 
(1 row)

SELECT n, ww_rows('ww', q) = ww_rows('wn', q) AS same_as_narrow FROM ww_q ORDER BY n;
 n  | same_as_narrow 
----+----------------
  1 | t
  2 | t
  3 | t
  4 | t
  5 | t
  6 | t
  7 | t
  8 | t
  9 | t
 10 | t
(10 rows)

-- promoted: the narrow buffer's segments and a wide buffer opened after them
SELECT sift.promote('wn_body'), sift.promote('ww_body');
 promote | promote 
---------+---------
       0 |       0
(1 row)

INSERT INTO ww SELECT i, ww_doc(i) FROM generate_series(2001, 2400) i;
INSERT INTO wn SELECT i, ww_doc(i) FROM generate_series(2001, 2400) i;
SELECT format FROM sift._test_wbuf_reads('wn_body');
 format 
--------
      6
(1 row)

SELECT n, ww_rows('ww', q) = ww_rows('wn', q) AS same_as_narrow FROM ww_q ORDER BY n;
 n  | same_as_narrow 
----+----------------
  1 | t
  2 | t
  3 | t
  4 | t
  5 | t
  6 | t
  7 | t
  8 | t
  9 | t
 10 | t
(10 rows)

SELECT sift.verify('ww_body'), sift.verify('wn_body');
 verify | verify 
--------+--------
        | 
(1 row)

DROP FUNCTION ww_doc(int), ww_walk(text, text), ww_rows(text, text);
DROP TABLE ww, wn, ww_q;
DROP EXTENSION pg_sift_test;
DROP EXTENSION pg_sift;