// Copyright (c) 2023-2025 ParadeDB, Inc.
//
// This file is part of ParadeDB - Postgres for Search and Analytics
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
#![allow(unused_variables, unused_imports)]
mod fixtures;
use std::path::PathBuf;
use fixtures::*;
use pretty_assertions::assert_eq;
use rstest::*;
use sqlx::PgConnection;
fn fmt_err(err: T) -> String {
format!("unexpected error, received: {}", err)
}
#[rstest]
fn invalid_create_index(mut conn: PgConnection) {
"CALL paradedb.create_bm25_test_table(table_name => 'index_config', schema_name => 'public')"
.execute(&mut conn);
match r#"CREATE INDEX index_config_index ON index_config
USING bm25 (id) "#
.execute_result(&mut conn)
{
Ok(_) => panic!("should fail with no key_field"),
Err(err) => assert_eq!(
err.to_string(),
"error returned from database: key_field WITH option should be configured"
),
};
}
#[rstest]
fn prevent_duplicate(mut conn: PgConnection) {
"CALL paradedb.create_bm25_test_table(table_name => 'index_config', schema_name => 'paradedb')"
.execute(&mut conn);
r#"CREATE INDEX index_config_index ON paradedb.index_config
USING bm25 (id, description) WITH (key_field='id')"#
.execute(&mut conn);
match r#"CREATE INDEX index_config_index ON paradedb.index_config
USING bm25 (id, description) WITH (key_field='id')"#
.execute_result(&mut conn)
{
Ok(_) => panic!("should fail with relation already exists"),
Err(err) => assert!(
err.to_string().contains("already exists"),
"{}",
fmt_err(err)
),
};
}
#[rstest]
async fn drop_column(mut conn: PgConnection) {
r#"
CREATE TABLE f_table (
id SERIAL PRIMARY KEY,
category TEXT
);
CREATE TABLE test_table (
id SERIAL PRIMARY KEY,
fkey INTEGER REFERENCES f_table ON UPDATE CASCADE ON DELETE RESTRICT,
fulltext TEXT
);
INSERT INTO f_table (category) VALUES ('cat_a'), ('cat_b'), ('cat_c');
INSERT INTO test_table (fkey, fulltext) VALUES (1, 'abc'), (1, 'def'), (2, 'ghi'), (3, 'jkl');
"#
.execute(&mut conn);
r#"CREATE INDEX test_index ON test_table
USING bm25 (id, fulltext) WITH (key_field='id')"#
.execute(&mut conn);
r#"DROP INDEX test_index CASCADE;
ALTER TABLE test_table DROP COLUMN fkey;
CREATE INDEX test_index ON test_table
USING bm25 (id, fulltext) WITH (key_field='id')"#
.execute(&mut conn);
let rows: Vec<(String, String)> =
"SELECT name, field_type FROM paradedb.schema('test_index')".fetch(&mut conn);
assert_eq!(rows[0], ("ctid".into(), "U64".into()));
assert_eq!(rows[1], ("fulltext".into(), "Str".into()));
assert_eq!(rows[2], ("id".into(), "I64".into()));
}
#[rstest]
fn default_text_field(mut conn: PgConnection) {
"CALL paradedb.create_bm25_test_table(table_name => 'index_config', schema_name => 'paradedb')"
.execute(&mut conn);
r#"CREATE INDEX index_config_index ON paradedb.index_config
USING bm25 (id, description) WITH (key_field='id')"#
.execute(&mut conn);
let rows: Vec<(String, String)> =
"SELECT name, field_type FROM paradedb.schema('paradedb.index_config_index')"
.fetch(&mut conn);
assert_eq!(rows[0], ("ctid".into(), "U64".into()));
assert_eq!(rows[1], ("description".into(), "Str".into()));
assert_eq!(rows[2], ("id".into(), "I64".into()));
}
#[rstest]
fn text_field_with_options(mut conn: PgConnection) {
"CALL paradedb.create_bm25_test_table(table_name => 'index_config', schema_name => 'paradedb')"
.execute(&mut conn);
r#"CREATE INDEX index_config_index ON paradedb.index_config
USING bm25 (id, description)
WITH (key_field='id', text_fields='{"description": {"tokenizer": {"type": "en_stem", "normalizer": "raw"}, "record": "freq", "fast": true}}');
"#
.execute(&mut conn);
let rows: Vec<(String, String)> =
"SELECT name, field_type FROM paradedb.schema('paradedb.index_config_index')"
.fetch(&mut conn);
assert_eq!(rows[0], ("ctid".into(), "U64".into()));
assert_eq!(rows[1], ("description".into(), "Str".into()));
assert_eq!(rows[2], ("id".into(), "I64".into()));
}
#[rstest]
fn multiple_text_fields(mut conn: PgConnection) {
"CALL paradedb.create_bm25_test_table(table_name => 'index_config', schema_name => 'paradedb')"
.execute(&mut conn);
r#"CREATE INDEX index_config_index ON paradedb.index_config
USING bm25 (id, description, category)
WITH (
key_field='id',
text_fields='{"description": {"tokenizer": {"type": "en_stem", "normalizer": "raw"}, "record": "freq", "fast": true}}'
);
"#
.execute(&mut conn);
let rows: Vec<(String, String)> =
"SELECT name, field_type FROM paradedb.schema('paradedb.index_config_index')"
.fetch(&mut conn);
assert_eq!(rows[0], ("category".into(), "Str".into()));
assert_eq!(rows[1], ("ctid".into(), "U64".into()));
assert_eq!(rows[2], ("description".into(), "Str".into()));
assert_eq!(rows[3], ("id".into(), "I64".into()));
}
#[rstest]
fn default_numeric_field(mut conn: PgConnection) {
"CALL paradedb.create_bm25_test_table(table_name => 'index_config', schema_name => 'paradedb')"
.execute(&mut conn);
r#"CREATE INDEX index_config_index ON paradedb.index_config
USING bm25 (id, rating) WITH (key_field='id')"#
.execute(&mut conn);
let rows: Vec<(String, String)> =
"SELECT name, field_type FROM paradedb.schema('paradedb.index_config_index')"
.fetch(&mut conn);
assert_eq!(rows[0], ("ctid".into(), "U64".into()));
assert_eq!(rows[1], ("id".into(), "I64".into()));
assert_eq!(rows[2], ("rating".into(), "I64".into()));
}
#[rstest]
fn numeric_field_with_options(mut conn: PgConnection) {
"CALL paradedb.create_bm25_test_table(table_name => 'index_config', schema_name => 'paradedb')"
.execute(&mut conn);
r#"CREATE INDEX index_config_index ON paradedb.index_config
USING bm25 (id, rating) WITH (key_field='id', numeric_fields='{"rating": {"fast": true}}')"#
.execute(&mut conn);
let rows: Vec<(String, String)> =
"SELECT name, field_type FROM paradedb.schema('paradedb.index_config_index')"
.fetch(&mut conn);
assert_eq!(rows[0], ("ctid".into(), "U64".into()));
assert_eq!(rows[1], ("id".into(), "I64".into()));
assert_eq!(rows[2], ("rating".into(), "I64".into()));
}
#[rstest]
fn default_boolean_field(mut conn: PgConnection) {
"CALL paradedb.create_bm25_test_table(table_name => 'index_config', schema_name => 'paradedb')"
.execute(&mut conn);
r#"CREATE INDEX index_config_index ON paradedb.index_config
USING bm25 (id, in_stock) WITH (key_field='id')"#
.execute(&mut conn);
let rows: Vec<(String, String)> =
"SELECT name, field_type FROM paradedb.schema('paradedb.index_config_index')"
.fetch(&mut conn);
assert_eq!(rows[0], ("ctid".into(), "U64".into()));
assert_eq!(rows[1], ("id".into(), "I64".into()));
assert_eq!(rows[2], ("in_stock".into(), "Bool".into()));
}
#[rstest]
fn boolean_field_with_options(mut conn: PgConnection) {
"CALL paradedb.create_bm25_test_table(table_name => 'index_config', schema_name => 'paradedb')"
.execute(&mut conn);
r#"CREATE INDEX index_config_index ON paradedb.index_config
USING bm25 (id, in_stock) WITH (key_field='id', boolean_fields='{"in_stock": {"fast": false}}')"#
.execute(&mut conn);
let rows: Vec<(String, String)> =
"SELECT name, field_type FROM paradedb.schema('paradedb.index_config_index')"
.fetch(&mut conn);
assert_eq!(rows[0], ("ctid".into(), "U64".into()));
assert_eq!(rows[1], ("id".into(), "I64".into()));
assert_eq!(rows[2], ("in_stock".into(), "Bool".into()));
}
#[rstest]
fn default_json_field(mut conn: PgConnection) {
"CALL paradedb.create_bm25_test_table(table_name => 'index_config', schema_name => 'paradedb')"
.execute(&mut conn);
r#"CREATE INDEX index_config_index ON paradedb.index_config
USING bm25 (id, metadata) WITH (key_field='id')"#
.execute(&mut conn);
let rows: Vec<(String, String)> =
"SELECT name, field_type FROM paradedb.schema('paradedb.index_config_index')"
.fetch(&mut conn);
assert_eq!(rows[0], ("ctid".into(), "U64".into()));
assert_eq!(rows[1], ("id".into(), "I64".into()));
assert_eq!(rows[2], ("metadata".into(), "JsonObject".into()));
}
#[rstest]
fn json_field_with_options(mut conn: PgConnection) {
"CALL paradedb.create_bm25_test_table(table_name => 'index_config', schema_name => 'paradedb')"
.execute(&mut conn);
r#"CREATE INDEX index_config_index ON paradedb.index_config
USING bm25 (id, metadata)
WITH (
key_field='id',
json_fields='{"metadata": {"fast": true, "expand_dots": false, "tokenizer": {"type": "raw", "normalizer": "raw"}}}'
)"#
.execute(&mut conn);
let rows: Vec<(String, String)> =
"SELECT name, field_type FROM paradedb.schema('paradedb.index_config_index')"
.fetch(&mut conn);
assert_eq!(rows[0], ("ctid".into(), "U64".into()));
assert_eq!(rows[1], ("id".into(), "I64".into()));
assert_eq!(rows[2], ("metadata".into(), "JsonObject".into()));
}
#[rstest]
fn default_datetime_field(mut conn: PgConnection) {
"CALL paradedb.create_bm25_test_table(table_name => 'index_config', schema_name => 'paradedb')"
.execute(&mut conn);
r#"CREATE INDEX index_config_index ON paradedb.index_config
USING bm25 (id, created_at, last_updated_date) WITH (key_field='id')"#
.execute(&mut conn);
let rows: Vec<(String, String)> =
"SELECT name, field_type FROM paradedb.schema('paradedb.index_config_index')"
.fetch(&mut conn);
assert_eq!(rows[0], ("created_at".into(), "Date".into()));
assert_eq!(rows[1], ("ctid".into(), "U64".into()));
assert_eq!(rows[2], ("id".into(), "I64".into()));
assert_eq!(rows[3], ("last_updated_date".into(), "Date".into()));
}
#[rstest]
fn datetime_field_with_options(mut conn: PgConnection) {
"CALL paradedb.create_bm25_test_table(table_name => 'index_config', schema_name => 'paradedb')"
.execute(&mut conn);
r#"CREATE INDEX index_config_index ON paradedb.index_config
USING bm25 (id, created_at, last_updated_date)
WITH (key_field='id', datetime_fields='{"created_at": {"fast": true}, "last_updated_date": {"fast": false}}')"#
.execute(&mut conn);
let rows: Vec<(String, String)> =
"SELECT name, field_type FROM paradedb.schema('paradedb.index_config_index')"
.fetch(&mut conn);
assert_eq!(rows[0], ("created_at".into(), "Date".into()));
assert_eq!(rows[1], ("ctid".into(), "U64".into()));
assert_eq!(rows[2], ("id".into(), "I64".into()));
assert_eq!(rows[3], ("last_updated_date".into(), "Date".into()));
}
#[rstest]
fn multiple_fields(mut conn: PgConnection) {
"CALL paradedb.create_bm25_test_table(table_name => 'index_config', schema_name => 'paradedb')"
.execute(&mut conn);
r#"CREATE INDEX index_config_index ON paradedb.index_config
USING bm25 (id, description, category, rating, in_stock, metadata) WITH (key_field='id')"#
.execute(&mut conn);
let rows: Vec<(String, String)> =
"SELECT name, field_type FROM paradedb.schema('paradedb.index_config_index')"
.fetch(&mut conn);
assert_eq!(rows[0], ("category".into(), "Str".into()));
assert_eq!(rows[1], ("ctid".into(), "U64".into()));
assert_eq!(rows[2], ("description".into(), "Str".into()));
assert_eq!(rows[3], ("id".into(), "I64".into()));
assert_eq!(rows[4], ("in_stock".into(), "Bool".into()));
assert_eq!(rows[5], ("metadata".into(), "JsonObject".into()));
assert_eq!(rows[6], ("rating".into(), "I64".into()));
}
#[rstest]
fn missing_schema_index(mut conn: PgConnection) {
match "SELECT paradedb.schema('paradedb.missing_bm25_index')".fetch_result::<(i64,)>(&mut conn)
{
Err(err) => assert!(err
.to_string()
.contains(r#"relation "paradedb.missing_bm25_index" does not exist"#)),
_ => panic!("non-existing index should throw an error"),
}
}
#[rstest]
fn null_values(mut conn: PgConnection) {
"CALL paradedb.create_bm25_test_table(table_name => 'index_config', schema_name => 'paradedb')"
.execute(&mut conn);
"INSERT INTO paradedb.index_config (description, category, rating) VALUES ('Null Item 1', NULL, NULL), ('Null Item 2', NULL, 2)"
.execute(&mut conn);
r#"CREATE INDEX index_config_index ON paradedb.index_config
USING bm25 (id, description, category, rating, in_stock, metadata) WITH (key_field='id')"#
.execute(&mut conn);
let rows: Vec<(String, Option, Option)> = "
SELECT description, category, rating
FROM paradedb.index_config WHERE index_config @@@ 'description:\"Null Item\"'
ORDER BY id"
.fetch(&mut conn);
assert_eq!(rows.len(), 2);
assert_eq!(rows[0], ("Null Item 1".into(), None, None));
assert_eq!(rows[1], ("Null Item 2".into(), None, Some(2)));
let rows: Vec<(bool,)> =
"SELECT in_stock FROM paradedb.index_config WHERE index_config @@@ 'in_stock:false'"
.fetch(&mut conn);
assert_eq!(rows.len(), 13);
}
#[rstest]
fn null_key_field_build(mut conn: PgConnection) {
"CREATE TABLE paradedb.index_config(id INTEGER, description TEXT)".execute(&mut conn);
"INSERT INTO paradedb.index_config VALUES (NULL, 'Null Item 1'), (2, 'Null Item 2')"
.execute(&mut conn);
match r#"CREATE INDEX index_config_index ON paradedb.index_config
USING bm25 (id, description) WITH (key_field='id')"#
.execute_result(&mut conn)
{
Ok(_) => panic!("should fail with null key_field"),
Err(err) => assert_eq!(
err.to_string(),
"error returned from database: error creating index entries for index 'index_config_index': key_field column 'id' cannot be NULL"
),
};
}
#[rstest]
fn null_key_field_insert(mut conn: PgConnection) {
"CREATE TABLE paradedb.index_config(id INTEGER, description TEXT)".execute(&mut conn);
"INSERT INTO paradedb.index_config VALUES (1, 'Null Item 1'), (2, 'Null Item 2')"
.execute(&mut conn);
r#"CREATE INDEX index_config_index ON paradedb.index_config
USING bm25 (id, description) WITH (key_field='id')"#
.execute(&mut conn);
match "INSERT INTO paradedb.index_config VALUES (NULL, 'Null Item 3')".execute_result(&mut conn)
{
Ok(_) => panic!("should fail with null key_field"),
Err(err) => assert_eq!(
err.to_string(),
"error returned from database: error creating index entries for index 'index_config_index': key_field column 'id' cannot be NULL"
),
};
}
#[rstest]
fn column_name_camelcase(mut conn: PgConnection) {
"CREATE TABLE paradedb.index_config(\"IdName\" INTEGER, \"ColumnName\" TEXT)"
.execute(&mut conn);
"INSERT INTO paradedb.index_config VALUES (1, 'Plastic Keyboard'), (2, 'Bluetooth Headphones')"
.execute(&mut conn);
r#"CREATE INDEX index_config_index ON paradedb.index_config
USING bm25 ("IdName", "ColumnName") WITH (key_field='IdName')"#
.execute(&mut conn);
let rows: Vec<(i32, String)> =
"SELECT * FROM paradedb.index_config WHERE index_config @@@ 'ColumnName:keyboard'"
.fetch(&mut conn);
assert_eq!(rows.len(), 1);
assert_eq!(rows[0], (1, "Plastic Keyboard".into()));
}
#[rstest]
fn multi_index_insert_in_transaction(mut conn: PgConnection) {
"CREATE TABLE paradedb.index_config1(id INTEGER, description TEXT)".execute(&mut conn);
"CREATE TABLE paradedb.index_config2(id INTEGER, description TEXT)".execute(&mut conn);
r#"CREATE INDEX index_config1_index ON paradedb.index_config1
USING bm25 (id, description) WITH (key_field='id')"#
.execute(&mut conn);
r#"CREATE INDEX index_config2_index ON paradedb.index_config2
USING bm25 (id, description) WITH (key_field='id')"#
.execute(&mut conn);
"BEGIN".execute(&mut conn);
"INSERT INTO paradedb.index_config1 VALUES (1, 'Item 1'), (2, 'Item 2')".execute(&mut conn);
"INSERT INTO paradedb.index_config2 VALUES (1, 'Item 1'), (2, 'Item 2')".execute(&mut conn);
"COMMIT".execute(&mut conn);
let rows: Vec<(i32, String)> =
"SELECT * FROM paradedb.index_config1 WHERE index_config1 @@@ 'description:item'"
.fetch(&mut conn);
assert_eq!(rows.len(), 2);
let rows: Vec<(i32, String)> =
"SELECT * FROM paradedb.index_config2 WHERE index_config2 @@@ 'description:item'"
.fetch(&mut conn);
assert_eq!(rows.len(), 2);
}
#[rstest]
fn partitioned_index(mut conn: PgConnection) {
// Set up the partitioned table with two partitions
r#"
CREATE TABLE sales (
id SERIAL,
sale_date DATE NOT NULL,
amount real NOT NULL, description TEXT,
PRIMARY KEY (id, sale_date)
) PARTITION BY RANGE (sale_date);
CREATE TABLE sales_2023_q1 PARTITION OF sales
FOR VALUES FROM ('2023-01-01') TO ('2023-03-31');
CREATE TABLE sales_2023_q2 PARTITION OF sales
FOR VALUES FROM ('2023-04-01') TO ('2023-06-30');
INSERT INTO sales (sale_date, amount, description) VALUES
('2023-01-10', 150.00, 'Ergonomic metal keyboard'),
('2023-01-15', 200.00, 'Plastic keyboard'),
('2023-02-05', 300.00, 'Sleek running shoes'),
('2023-03-12', 175.50, 'Bluetooth speaker'),
('2023-03-25', 225.75, 'Artistic ceramic vase');
INSERT INTO sales (sale_date, amount, description) VALUES
('2023-04-01', 250.00, 'Modern wall clock'),
('2023-04-18', 180.00, 'Designer wall paintings'),
('2023-05-09', 320.00, 'Handcrafted wooden frame');
"#
.execute(&mut conn);
// Create the BM25 index on the partitioned table
r#"CREATE INDEX sales_index ON sales_2023_q1
USING bm25 (id, description, sale_date, amount) WITH (key_field='id', numeric_fields='{"amount": {"fast": true}}')
"#
.execute(&mut conn);
// Test: Verify data is partitioned correctly by querying each partition
let rows_q1: Vec<(i32, String, String)> = r#"
SELECT id, description, sale_date::text FROM sales_2023_q1
"#
.fetch(&mut conn);
assert_eq!(rows_q1.len(), 5, "Expected 5 rows in Q1 partition");
let rows_q2: Vec<(i32, String, String)> = r#"
SELECT id, description, sale_date::text FROM sales_2023_q2
"#
.fetch(&mut conn);
assert_eq!(rows_q2.len(), 3, "Expected 3 rows in Q2 partition");
// Test: Search using the bm25 index
let search_results: Vec<(i32, String)> = r#"
SELECT id, description FROM sales_2023_q1 WHERE id @@@ 'description:keyboard'
"#
.fetch(&mut conn);
assert_eq!(search_results.len(), 2, "Expected 2 items with 'keyboard'");
// Test: Retrieve items by a numeric range (amount field) and verify bm25 compatibility
let amount_results: Vec<(i32, String, f32)> = r#"
SELECT id, description, amount FROM sales_2023_q1
WHERE amount @@@ '[175 TO 250]'
ORDER BY amount ASC
"#
.fetch(&mut conn);
assert_eq!(
amount_results.len(),
3,
"Expected 3 items with amount in range 175-250"
);
}
#[rstest]
fn custom_enum_term(mut conn: PgConnection) {
r#"
CREATE TYPE color AS ENUM ('red', 'green', 'blue');
CREATE TABLE paradedb.index_config(id INTEGER, description TEXT, color color);
INSERT INTO paradedb.index_config VALUES (1, 'Item 1', 'red'), (2, 'Item 2', 'green');
"#
.execute(&mut conn);
r#"
CREATE INDEX index_config_index ON paradedb.index_config
USING bm25 (id, description, color)
WITH (key_field='id');
"#
.execute(&mut conn);
let rows: Vec<(i32, String)> =
"SELECT id, description FROM paradedb.index_config WHERE id @@@ paradedb.term('color', 'red'::color)".fetch(&mut conn);
assert_eq!(rows, vec![(1, "Item 1".into())]);
}
#[rstest]
fn custom_enum_parse(mut conn: PgConnection) {
r#"
CREATE TYPE color AS ENUM ('red', 'green', 'blue');
CREATE TABLE paradedb.index_config(id INTEGER, description TEXT, color color);
INSERT INTO paradedb.index_config VALUES (1, 'Item 1', 'red'), (2, 'Item 2', 'green');
"#
.execute(&mut conn);
r#"
CREATE INDEX index_config_index ON paradedb.index_config
USING bm25 (id, description, color)
WITH (key_field='id');
"#
.execute(&mut conn);
let rows: Vec<(i32, String)> =
"SELECT id, description FROM paradedb.index_config WHERE id @@@ paradedb.parse('color:1.0')".fetch(&mut conn);
assert_eq!(rows, vec![(1, "Item 1".into())]);
}
#[rstest]
fn long_text_key_field_issue2198(mut conn: PgConnection) {
"CREATE TABLE issue2198 (id TEXT, value TEXT)".execute(&mut conn);
"CREATE INDEX idxissue2198 ON issue2198 USING bm25 (id, value) WITH (key_field='id')"
.execute(&mut conn);
let long_string = "a".repeat(10000);
format!("INSERT INTO issue2198(id) VALUES ('{long_string}')").execute(&mut conn);
let (count,) = format!("SELECT count(*) FROM issue2198 WHERE id @@@ '{long_string}'")
.fetch_one::<(i64,)>(&mut conn);
assert_eq!(count, 1);
let (count,) =
format!("SELECT count(*) FROM issue2198 WHERE id @@@ paradedb.term('id', '{long_string}')")
.fetch_one::<(i64,)>(&mut conn);
assert_eq!(count, 1);
}
#[rstest]
fn uuid_as_raw_issue2199(mut conn: PgConnection) {
"CREATE TABLE issue2199 (id SERIAL8 NOT NULL PRIMARY KEY, value uuid);".execute(&mut conn);
"CREATE INDEX idxissue2199 ON issue2199 USING bm25 (id, value) WITH (key_field='id');"
.execute(&mut conn);
let uuid = uuid::Uuid::new_v4();
format!("INSERT INTO issue2199(value) VALUES ('{uuid}')").execute(&mut conn);
let (count,) = format!("SELECT count(*) FROM issue2199 WHERE value @@@ '{uuid}'")
.fetch_one::<(i64,)>(&mut conn);
assert_eq!(count, 1);
let (count,) =
format!("SELECT count(*) FROM issue2199 WHERE id @@@ paradedb.term('value', '{uuid}')")
.fetch_one::<(i64,)>(&mut conn);
assert_eq!(count, 1);
}