-- test stopwords and stopwords_language
\echo 'Test: Stopwords processing'
Test: Stopwords processing
-- direct stop words list
SELECT * FROM paradedb.tokenize(
        paradedb.tokenizer('default', stopwords => ARRAY['stopword']),
        'something, stopword, else'
);
   token   | position 
-----------+----------
 something |        0
 else      |        2
(2 rows)

SELECT * FROM paradedb.tokenize(
        paradedb.tokenizer('default', stopwords_language => 'English'),
        'something and else'
);
   token   | position 
-----------+----------
 something |        0
 else      |        2
(2 rows)

-- direct stopwords AND stopwords language
SELECT * FROM paradedb.tokenize(
        paradedb.tokenizer('default', stopwords_language => 'English', stopwords => ARRAY['stopword']),
        'stopword and else'
);
 token | position 
-------+----------
 else  |        2
(1 row)

-- Use in search
DROP TABLE IF EXISTS test_stopwords CASCADE;
CREATE TABLE test_stopwords
(
    id    serial8 not null primary key,
    name  text
);
insert into test_stopwords (name)
values
    ('something, stopword, else'), -- those two should be equivalent with the index below
    ('something else'),
    ('something more');
CREATE INDEX idx_stopwords_bm25 ON test_stopwords
    USING bm25 (id, name)
    WITH (
    key_field = 'id',
    text_fields ='{
        "name": {"tokenizer": {"type": "default", "stopwords": ["stopword"]}}
    }'
);
-- "something else" and "something, stopword, else" have the same score
SELECT paradedb.score(id) AS score, name
FROM test_stopwords
WHERE name @@@ '("something" "else")'
ORDER BY name;
   score    |           name            
------------+---------------------------
 0.60353506 | something else
 0.13353144 | something more
 0.60353506 | something, stopword, else
(3 rows)

-- stopword is filtered out hence when trying to search for it, the result is empty
SELECT paradedb.score(id) AS score, name
FROM test_stopwords
WHERE name @@@ 'and'
ORDER BY name;
 score | name 
-------+------
(0 rows)

DROP TABLE test_stopwords CASCADE;
--- Languages support:
SELECT * FROM paradedb.tokenize(
        paradedb.tokenizer('default', stopwords_language => 'Danish'),
        'ikke æbler'
              );
 token | position 
-------+----------
 æbler |        1
(1 row)

SELECT * FROM paradedb.tokenize(
        paradedb.tokenizer('default', stopwords_language => 'Dutch'),
        'geen appels'
              );
 token  | position 
--------+----------
 appels |        1
(1 row)

SELECT * FROM paradedb.tokenize(
        paradedb.tokenizer('default', stopwords_language => 'English'),
        'no apples'
              );
 token  | position 
--------+----------
 apples |        1
(1 row)

SELECT * FROM paradedb.tokenize(
        paradedb.tokenizer('default', stopwords_language => 'Finnish'),
        'ei omenoita'
              );
  token   | position 
----------+----------
 omenoita |        1
(1 row)

SELECT * FROM paradedb.tokenize(
        paradedb.tokenizer('default', stopwords_language => 'French'),
        'pas de pommes'
              );
 token  | position 
--------+----------
 pommes |        2
(1 row)

SELECT * FROM paradedb.tokenize(
        paradedb.tokenizer('default', stopwords_language => 'German'),
        'keine Äpfel'
              );
 token | position 
-------+----------
 äpfel |        1
(1 row)

SELECT * FROM paradedb.tokenize(
        paradedb.tokenizer('default', stopwords_language => 'Hungarian'),
        'nincs alma'
              );
 token | position 
-------+----------
 alma  |        1
(1 row)

SELECT * FROM paradedb.tokenize(
        paradedb.tokenizer('default', stopwords_language => 'Italian'),
        'non mele'
              );
 token | position 
-------+----------
 mele  |        1
(1 row)

SELECT * FROM paradedb.tokenize(
        paradedb.tokenizer('default', stopwords_language => 'Norwegian'),
        'ingen epler'
              );
 token | position 
-------+----------
 epler |        1
(1 row)

SELECT * FROM paradedb.tokenize(
        paradedb.tokenizer('default', stopwords_language => 'Portuguese'),
        'sem maçãs'
              );
 token | position 
-------+----------
 maçãs |        1
(1 row)

SELECT * FROM paradedb.tokenize(
        paradedb.tokenizer('default', stopwords_language => 'Russian'),
        'нет яблок'
              );
 token | position 
-------+----------
 яблок |        1
(1 row)

SELECT * FROM paradedb.tokenize(
        paradedb.tokenizer('default', stopwords_language => 'Spanish'),
        'sin manzanas'
              );
  token   | position 
----------+----------
 manzanas |        1
(1 row)

SELECT * FROM paradedb.tokenize(
        paradedb.tokenizer('default', stopwords_language => 'Swedish'),
        'inte äpplen'
              );
 token  | position 
--------+----------
 äpplen |        1
(1 row)