-- SQL_ASCII: bytes >= 0x80 are opaque letters, ASCII still folds
SELECT current_database() AS orig \gset
CREATE DATABASE sift_sql_ascii ENCODING 'SQL_ASCII' LOCALE 'C' TEMPLATE template0;
CREATE DATABASE sift_latin1 ENCODING 'LATIN1' LOCALE 'C' TEMPLATE template0;
\c sift_sql_ascii
CREATE EXTENSION pg_sift;
SELECT token::bytea, pos, start_offset, end_offset
  FROM sift.tokenize(E'Caf\xe9 NA\xcfVE \xff\xfe-x \xc3\xa9T\xc3\xa9');
    token     | pos | start_offset | end_offset 
--------------+-----+--------------+------------
 \x636166e9   |   0 |            0 |          4
 \x6e61cf7665 |   1 |            5 |         10
 \xfffe       |   2 |           11 |         13
 \x78         |   3 |           14 |         15
 \xc3a974c3a9 |   4 |           16 |         21
(5 rows)

SELECT octet_length(token) FROM sift.tokenize(repeat(E'\xe9', 300));
 octet_length 
--------------
          255
(1 row)

\c sift_latin1
CREATE EXTENSION pg_sift;
SELECT * FROM sift.tokenize('x');
ERROR:  sift requires a UTF8 or SQL_ASCII database, not LATIN1
HINT:  Create a database with ENCODING 'UTF8' and restore the data into it.
\c :orig
DROP DATABASE sift_sql_ascii;
DROP DATABASE sift_latin1;