// Copyright (c) 2023-2025 ParadeDB, Inc. // // This file is part of ParadeDB - Postgres for Search and Analytics // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . // Tests for ParadeDB's Custom Scan implementation mod fixtures; use fixtures::*; use pretty_assertions::assert_eq; use rstest::*; use serde_json::{Number, Value}; use sqlx::PgConnection; #[rstest] fn corrupt_targetlist(mut conn: PgConnection) { SimpleProductsTable::setup().execute(&mut conn); let (id, score) = "select count(*), max(paradedb.score(id)) from paradedb.bm25_search where description @@@ 'keyboard'" .fetch_one::<(i64,f32)>(&mut conn); assert_eq!((id, score), (2, 3.2668595)); "PREPARE prep AS select count(*), max(paradedb.score(id)) from paradedb.bm25_search where description @@@ 'keyboard'".execute(&mut conn); for _ in 0..100 { "EXECUTE prep".fetch_one::<(i64, f32)>(&mut conn); assert_eq!((id, score), (2, 3.2668595)); } } #[rstest] fn attribute_1_of_table_has_wrong_type(mut conn: PgConnection) { SimpleProductsTable::setup().execute(&mut conn); let (id, ) = "SELECT id, description FROM paradedb.bm25_search WHERE description @@@ 'keyboard' OR id = 1 ORDER BY id LIMIT 1" .fetch_one::<(i32,)>(&mut conn); assert_eq!(id, 1); } #[rstest] fn generates_custom_scan_for_or(mut conn: PgConnection) { use serde_json::Value; SimpleProductsTable::setup().execute(&mut conn); let (plan, ) = "EXPLAIN (ANALYZE, FORMAT JSON) SELECT * FROM paradedb.bm25_search WHERE bm25_search @@@ 'description:keyboard' OR description @@@ 'shoes'".fetch_one::<(Value,)>(&mut conn); let plan = plan .get(0) .unwrap() .as_object() .unwrap() .get("Plan") .unwrap() .as_object() .unwrap() .get("Plans") .unwrap() .get(0) .unwrap(); eprintln!("{plan:#?}"); assert_eq!( plan.get("Custom Plan Provider"), Some(&Value::String(String::from("ParadeDB Scan"))) ); } #[rstest] fn generates_custom_scan_for_and(mut conn: PgConnection) { use serde_json::Value; SimpleProductsTable::setup().execute(&mut conn); "SET enable_indexscan TO off;".execute(&mut conn); let (plan, ) = "EXPLAIN (ANALYZE, FORMAT JSON) SELECT * FROM paradedb.bm25_search WHERE bm25_search @@@ 'description:keyboard' AND description @@@ 'shoes'".fetch_one::<(Value,)>(&mut conn); let plan = plan.pointer("/0/Plan/Plans/0").unwrap(); eprintln!("{plan:#?}"); assert_eq!( plan.get("Custom Plan Provider"), Some(&Value::String(String::from("ParadeDB Scan"))) ); } #[rstest] fn includes_segment_count(mut conn: PgConnection) { use serde_json::Value; SimpleProductsTable::setup().execute(&mut conn); "SET enable_indexscan TO off;".execute(&mut conn); let (plan, ) = "EXPLAIN (ANALYZE, FORMAT JSON) SELECT * FROM paradedb.bm25_search WHERE bm25_search @@@ 'description:keyboard' AND description @@@ 'shoes'".fetch_one::<(Value,)>(&mut conn); let plan = plan.pointer("/0/Plan/Plans/0").unwrap(); assert!(plan.get("Segment Count").is_some()); } #[rstest] fn field_on_left(mut conn: PgConnection) { SimpleProductsTable::setup().execute(&mut conn); let (id,) = "SELECT id FROM paradedb.bm25_search WHERE description @@@ 'keyboard' ORDER BY id ASC" .fetch_one::<(i32,)>(&mut conn); assert_eq!(id, 1); } #[rstest] fn table_on_left(mut conn: PgConnection) { SimpleProductsTable::setup().execute(&mut conn); let (id, ) = "SELECT id FROM paradedb.bm25_search WHERE bm25_search @@@ 'description:keyboard' ORDER BY id ASC" .fetch_one::<(i32,)>(&mut conn); assert_eq!(id, 1); } #[rstest] fn scores_project(mut conn: PgConnection) { SimpleProductsTable::setup().execute(&mut conn); let (id, score) = "SELECT id, paradedb.score(id) FROM paradedb.bm25_search WHERE description @@@ 'keyboard' ORDER BY paradedb.score(id) DESC LIMIT 1" .fetch_one::<(i32, f32)>(&mut conn); assert_eq!(id, 2); assert_eq!(score, 3.2668595); } #[rstest] fn snippets_project(mut conn: PgConnection) { SimpleProductsTable::setup().execute(&mut conn); let (id, snippet) = "SELECT id, paradedb.snippet(description) FROM paradedb.bm25_search WHERE description @@@ 'keyboard' ORDER BY paradedb.score(id) DESC LIMIT 1" .fetch_one::<(i32, String)>(&mut conn); assert_eq!(id, 2); assert_eq!(snippet, String::from("Plastic Keyboard")); } #[rstest] fn scores_and_snippets_project(mut conn: PgConnection) { SimpleProductsTable::setup().execute(&mut conn); let (id, score, snippet) = "SELECT id, paradedb.score(id), paradedb.snippet(description) FROM paradedb.bm25_search WHERE description @@@ 'keyboard' ORDER BY paradedb.score(id) DESC LIMIT 1" .fetch_one::<(i32, f32, String)>(&mut conn); assert_eq!(id, 2); assert_eq!(score, 3.2668595); assert_eq!(snippet, String::from("Plastic Keyboard")); } #[rstest] fn mingets(mut conn: PgConnection) { SimpleProductsTable::setup().execute(&mut conn); let (id, snippet) = "SELECT id, paradedb.snippet(description, '', '') FROM paradedb.bm25_search WHERE description @@@ 'teddy bear'" .fetch_one::<(i32, String)>(&mut conn); assert_eq!(id, 40); assert_eq!( snippet, String::from("Plush teddy bear") ); } #[rstest] fn scores_with_expressions(mut conn: PgConnection) { SimpleProductsTable::setup().execute(&mut conn); let result = r#" select id, description, paradedb.score(id), rating, paradedb.score(id) * rating /* testing this, specifically */ from paradedb.bm25_search where metadata @@@ 'color:white' order by 5 desc, score desc limit 1; "# .fetch_one::<(i32, String, f32, i32, f64)>(&mut conn); assert_eq!( result, ( 25, "Anti-aging serum".into(), 3.2455924, 4, 12.982369422912598 ) ); } #[rstest] fn limit_without_order_by(mut conn: PgConnection) { SimpleProductsTable::setup().execute(&mut conn); "SET enable_indexscan TO off;".execute(&mut conn); let (plan, ) = r#" explain (analyze, format json) select * from paradedb.bm25_search where metadata @@@ 'color:white' limit 1; "# .fetch_one::<(Value, )>(&mut conn); let path = plan.pointer("/0/Plan/Plans/0").unwrap(); assert_eq!( path.get("Node Type"), Some(&Value::String(String::from("Custom Scan"))) ); assert_eq!(path.get("Scores"), Some(&Value::Bool(false))); assert_eq!( path.get(" Top N Limit"), Some(&Value::Number(Number::from(1))) ); } #[rstest] fn score_and_limit_without_order_by(mut conn: PgConnection) { SimpleProductsTable::setup().execute(&mut conn); "SET enable_indexscan TO off;".execute(&mut conn); let (plan, ) = r#" explain (analyze, format json) select paradedb.score(id), * from paradedb.bm25_search where metadata @@@ 'color:white' limit 1; "# .fetch_one::<(Value, )>(&mut conn); let path = plan.pointer("/0/Plan/Plans/0").unwrap(); assert_eq!( path.get("Node Type"), Some(&Value::String(String::from("Custom Scan"))) ); assert_eq!(path.get("Scores"), Some(&Value::Bool(true))); assert_eq!( path.get(" Top N Limit"), Some(&Value::Number(Number::from(1))) ); } #[rstest] fn simple_join_with_scores_and_both_sides(mut conn: PgConnection) { SimpleProductsTable::setup().execute(&mut conn); let result = r#" select a.id, a.score, b.id, b.score from (select paradedb.score(id), * from paradedb.bm25_search) a inner join (select paradedb.score(id), * from paradedb.bm25_search) b on a.id = b.id where a.description @@@ 'bear' AND b.description @@@ 'teddy bear';"# .fetch_one::<(i32, f32, i32, f32)>(&mut conn); assert_eq!(result, (40, 3.3322046, 40, 6.664409)); } #[rstest] fn simple_join_with_scores_on_both_sides(mut conn: PgConnection) { SimpleProductsTable::setup().execute(&mut conn); let result = r#" select a.id, a.score, b.id, b.score from (select paradedb.score(id), * from paradedb.bm25_search) a inner join (select paradedb.score(id), * from paradedb.bm25_search) b on a.id = b.id where a.description @@@ 'bear' OR b.description @@@ 'teddy bear';"# .fetch_one::<(i32, f32, i32, f32)>(&mut conn); assert_eq!(result, (40, 3.3322046, 40, 6.664409)); } #[rstest] fn add_scores_across_joins_issue1753(mut conn: PgConnection) { r#" CALL paradedb.create_bm25_test_table(table_name => 'mock_items', schema_name => 'public'); CREATE INDEX search_idx ON mock_items USING bm25 (id, description, category, rating, in_stock, metadata, created_at, last_updated_date, latest_available_time) WITH (key_field='id'); CALL paradedb.create_bm25_test_table( schema_name => 'public', table_name => 'orders', table_type => 'Orders' ); ALTER TABLE orders ADD CONSTRAINT foreign_key_product_id FOREIGN KEY (product_id) REFERENCES mock_items(id); CREATE INDEX orders_idx ON orders USING bm25 (order_id, customer_name) WITH (key_field='order_id'); "#.execute(&mut conn); // this one doesn't plan a custom scan at all, so scores come back as NaN let result = " SELECT o.order_id, m.description, paradedb.score(o.order_id) + paradedb.score(m.id) as score FROM orders o JOIN mock_items m ON o.product_id = m.id WHERE o.customer_name @@@ 'Johnson' AND m.description @@@ 'shoes' ORDER BY order_id LIMIT 1" .fetch_one::<(i32, String, f32)>(&mut conn); assert_eq!(result, (3, "Sleek running shoes".into(), 5.406531)); } #[rstest] fn scores_survive_joins(mut conn: PgConnection) { r#" CALL paradedb.create_bm25_test_table(table_name => 'a', schema_name => 'public'); CALL paradedb.create_bm25_test_table(table_name => 'b', schema_name => 'public'); CALL paradedb.create_bm25_test_table(table_name => 'c', schema_name => 'public'); CREATE INDEX idxa ON a USING bm25 (id, description, category, rating, in_stock, metadata, created_at, last_updated_date, latest_available_time) WITH (key_field='id'); CREATE INDEX idxb ON b USING bm25 (id, description, category, rating, in_stock, metadata, created_at, last_updated_date, latest_available_time) WITH (key_field='id'); CREATE INDEX idxc ON c USING bm25 (id, description, category, rating, in_stock, metadata, created_at, last_updated_date, latest_available_time) WITH (key_field='id'); "#.execute(&mut conn); // this one doesn't plan a custom scan at all, so scores come back as NaN let result = r#" SELECT a.description, paradedb.score(a.id) FROM a join b on a.id = b.id join c on a.id = c.id WHERE a.description @@@ 'shoes' ORDER BY a.description;"# .fetch_result::<(String, f32)>(&mut conn) .expect("query failed"); assert_eq!( result, vec![ ("Generic shoes".into(), 2.8772602), ("Sleek running shoes".into(), 2.4849067), ("White jogging shoes".into(), 2.4849067), ] ); } #[rustfmt::skip] #[rstest] fn join_issue_1776(mut conn: PgConnection) { r#" CALL paradedb.create_bm25_test_table( schema_name => 'public', table_name => 'mock_items' ); CREATE INDEX search_idx ON mock_items USING bm25 (id, description, category, rating, in_stock, metadata, created_at) WITH (key_field='id'); CALL paradedb.create_bm25_test_table( schema_name => 'public', table_name => 'orders', table_type => 'Orders' ); ALTER TABLE orders ADD CONSTRAINT foreign_key_product_id FOREIGN KEY (product_id) REFERENCES mock_items(id); CREATE INDEX orders_idx ON orders USING bm25 (order_id, customer_name) WITH (key_field='order_id'); "# .execute(&mut conn); let results = r#" SELECT o.order_id, m.description, o.customer_name, paradedb.score(o.order_id) as orders_score, paradedb.score(m.id) as items_score FROM orders o JOIN mock_items m ON o.product_id = m.id WHERE o.customer_name @@@ 'Johnson' AND m.description @@@ 'shoes' OR m.description @@@ 'Smith' ORDER BY order_id LIMIT 5; "#.fetch_result::<(i32, String, String, f32, f32)>(&mut conn).expect("query failed"); assert_eq!(results[0], (3, "Sleek running shoes".into(), "Alice Johnson".into(), 2.9216242, 2.4849067)); assert_eq!(results[1], (6, "White jogging shoes".into(), "Alice Johnson".into(), 2.9216242, 2.4849067)); assert_eq!(results[2], (36,"White jogging shoes".into(), "Alice Johnson".into(), 2.9216242, 2.4849067)); } #[rustfmt::skip] #[rstest] fn join_issue_1826(mut conn: PgConnection) { r#" CALL paradedb.create_bm25_test_table( schema_name => 'public', table_name => 'mock_items' ); CREATE INDEX search_idx ON mock_items USING bm25 (id, description, category, rating, in_stock, metadata, created_at) WITH (key_field='id'); CALL paradedb.create_bm25_test_table( schema_name => 'public', table_name => 'orders', table_type => 'Orders' ); ALTER TABLE orders ADD CONSTRAINT foreign_key_product_id FOREIGN KEY (product_id) REFERENCES mock_items(id); CREATE INDEX orders_idx ON orders USING bm25 (order_id, customer_name) WITH (key_field='order_id'); "# .execute(&mut conn); let results = r#" SELECT o.order_id, m.description, o.customer_name, paradedb.score(o.order_id) as orders_score, paradedb.score(m.id) as items_score FROM orders o JOIN mock_items m ON o.product_id = m.id WHERE o.customer_name @@@ 'Johnson' AND m.description @@@ 'shoes' OR m.description @@@ 'Smith' ORDER BY paradedb.score(m.id) desc, m.id asc LIMIT 1; "#.fetch_result::<(i32, String, String, f32, f32)>(&mut conn).expect("query failed"); assert_eq!(results[0], (3, "Sleek running shoes".into(), "Alice Johnson".into(), 2.9216242, 2.4849067)); } #[rstest] fn leaky_file_handles(mut conn: PgConnection) { r#" CREATE OR REPLACE FUNCTION raise_exception(int, int) RETURNS bool LANGUAGE plpgsql AS $$ DECLARE BEGIN IF $1 = $2 THEN RAISE EXCEPTION 'error! % = %', $1, $2; END IF; RETURN false; END; $$; "# .execute(&mut conn); let (pid,) = "SELECT pg_backend_pid()".fetch_one::<(i32,)>(&mut conn); SimpleProductsTable::setup().execute(&mut conn); // this will raise an error when it hits id #12 let result = "SELECT id, paradedb.score(id), raise_exception(id, 12) FROM paradedb.bm25_search WHERE category @@@ 'electronics' ORDER BY paradedb.score(id) DESC, id LIMIT 10" .execute_result(&mut conn); assert!(result.is_err()); assert_eq!( "error returned from database: error! 12 = 12", &format!("{}", result.err().unwrap()) ); fn tantivy_files_still_open(pid: i32) -> bool { let output = std::process::Command::new("lsof") .arg("-p") .arg(pid.to_string()) .output() .expect("`lsof` command should not fail`"); let stdout = String::from_utf8_lossy(&output.stdout); eprintln!("stdout: {}", stdout); stdout.contains("/tantivy/") } // see if there's still some open tantivy files if tantivy_files_still_open(pid) { // if there are, they're probably (hopefully!) from where we the postgres connection // is waiting on merge threads in the background. So we'll give it 5 seconds and try again eprintln!("sleeping for 5s and checking open files again"); std::thread::sleep(std::time::Duration::from_secs(5)); // this time asserting for real assert!(!tantivy_files_still_open(pid)); } } #[rustfmt::skip] #[rstest] fn cte_issue_1951(mut conn: PgConnection) { r#" CREATE TABLE t ( id SERIAL, data TEXT ); CREATE TABLE s ( id SERIAL, data TEXT ); insert into t (id, data) select x, md5(x::text) || ' query' from generate_series(1, 100) x; insert into s (id, data) select x, md5(x::text) from generate_series(1, 100) x; create index idxt on t using bm25 (id, data) with (key_field = id); create index idxs on s using bm25 (id, data) with (key_field = id); "#.execute(&mut conn); let results = r#" with cte as ( select id, 1 as score from t where data @@@ 'query' limit 1) select cte.id from s right join cte on cte.id = s.id order by cte.score desc; "#.fetch_result::<(i32, )>(&mut conn).expect("query failed"); assert_eq!(results.len(), 1); } #[rstest] fn is_numeric_fast_field_capable(mut conn: PgConnection) { r#" CREATE TABLE test ( id SERIAL8 NOT NULL PRIMARY KEY, message TEXT, severity INTEGER ) WITH (autovacuum_enabled = false); INSERT INTO test (message, severity) VALUES ('beer wine cheese a', 1); INSERT INTO test (message, severity) VALUES ('beer wine a', 2); INSERT INTO test (message, severity) VALUES ('beer cheese a', 3); INSERT INTO test (message, severity) VALUES ('beer a', 4); INSERT INTO test (message, severity) VALUES ('wine cheese a', 5); INSERT INTO test (message, severity) VALUES ('wine a', 6); INSERT INTO test (message, severity) VALUES ('cheese a', 7); INSERT INTO test (message, severity) VALUES ('beer wine cheese a', 1); INSERT INTO test (message, severity) VALUES ('beer wine a', 2); INSERT INTO test (message, severity) VALUES ('beer cheese a', 3); INSERT INTO test (message, severity) VALUES ('beer a', 4); INSERT INTO test (message, severity) VALUES ('wine cheese a', 5); INSERT INTO test (message, severity) VALUES ('wine a', 6); INSERT INTO test (message, severity) VALUES ('cheese a', 7); -- INSERT INTO test (message) SELECT 'space fillter ' || x FROM generate_series(1, 10000000) x; CREATE INDEX idxtest ON test USING bm25(id, message, severity) WITH (key_field = 'id'); CREATE OR REPLACE FUNCTION assert(a bigint, b bigint) RETURNS bool STABLE STRICT LANGUAGE plpgsql AS $$ DECLARE current_txid bigint; BEGIN -- Get the current transaction ID current_txid := txid_current(); -- Check if the values are not equal IF a <> b THEN RAISE EXCEPTION 'Assertion failed: % <> %. Transaction ID: %', a, b, current_txid; END IF; RETURN true; END; $$; "#.execute(&mut conn); "VACUUM test;".execute(&mut conn); r#" SET enable_indexonlyscan to OFF; SET enable_indexscan to OFF; "# .execute(&mut conn); let (b, count) = "select assert(count(*), 8), count(*) from (select id from test where message @@@ 'beer' order by severity) x limit 8;".fetch_one::<(bool, i64)>(&mut conn); assert_eq!((b, count), (true, 8)); } #[rstest] fn top_n_matches(mut conn: PgConnection) { r#" DROP TABLE IF EXISTS test; CREATE TABLE test ( id SERIAL8 NOT NULL PRIMARY KEY, message TEXT, severity INTEGER ) WITH (autovacuum_enabled = false); INSERT INTO test (message, severity) VALUES ('beer wine cheese a', 1); INSERT INTO test (message, severity) VALUES ('beer wine a', 2); INSERT INTO test (message, severity) VALUES ('beer cheese a', 3); INSERT INTO test (message, severity) VALUES ('beer a', 4); INSERT INTO test (message, severity) VALUES ('wine cheese a', 5); INSERT INTO test (message, severity) VALUES ('wine a', 6); INSERT INTO test (message, severity) VALUES ('cheese a', 7); INSERT INTO test (message, severity) VALUES ('beer wine cheese a', 1); INSERT INTO test (message, severity) VALUES ('beer wine a', 2); INSERT INTO test (message, severity) VALUES ('beer cheese a', 3); INSERT INTO test (message, severity) VALUES ('beer a', 4); INSERT INTO test (message, severity) VALUES ('wine cheese a', 5); INSERT INTO test (message, severity) VALUES ('wine a', 6); INSERT INTO test (message, severity) VALUES ('cheese a', 7); -- INSERT INTO test (message) SELECT 'space fillter ' || x FROM generate_series(1, 10000000) x; CREATE INDEX idxtest ON test USING bm25(id, message, severity) WITH (key_field = 'id'); CREATE OR REPLACE FUNCTION assert(a bigint, b bigint) RETURNS bool STABLE STRICT LANGUAGE plpgsql AS $$ DECLARE current_txid bigint; BEGIN -- Get the current transaction ID current_txid := txid_current(); -- Check if the values are not equal IF a <> b THEN RAISE EXCEPTION 'Assertion failed: % <> %. Transaction ID: %', a, b, current_txid; END IF; RETURN true; END; $$; "#.execute(&mut conn); "UPDATE test SET severity = (floor(random() * 10) + 1)::int WHERE id < 10;".execute(&mut conn); "UPDATE test SET severity = (floor(random() * 10) + 1)::int WHERE id < 10;".execute(&mut conn); "UPDATE test SET severity = (floor(random() * 10) + 1)::int WHERE id < 10;".execute(&mut conn); r#" SET enable_indexonlyscan to OFF; SET enable_indexscan to OFF; "# .execute(&mut conn); for n in [1, 2, 3, 4, 5, 6, 7, 8, 100] { let sql = format!("select assert(count(*), LEAST({n}, 8)), count(*) from (select id from test where message @@@ 'beer' order by severity limit {n}) x;"); let (b, count) = sql.fetch_one::<(bool, i64)>(&mut conn); assert_eq!((b, count), (true, n.min(8))); } }